Skip to content

Commit

Permalink
Use _opt codec methods internally throughout the crate
Browse files Browse the repository at this point in the history
  • Loading branch information
@LDeakin
LDeakin committed Feb 12, 2024
1 parent b20da18 commit e2eb641
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 17 deletions.
8 changes: 6 additions & 2 deletions src/array/array_async_readable.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -571,11 +571,15 @@ impl<TStorage: ?Sized + AsyncReadableStorageTraits + 'static> Array<TStorage> {
self.chunk_array_representation(&chunk_indices)?;
let partial_decoder = self
.codecs()
.async_partial_decoder(input_handle, &chunk_representation)
.async_partial_decoder_opt(
input_handle,
&chunk_representation,
options, // FIXME: Adjust internal decode options
)

Check warning on line 578 in src/array/array_async_readable.rs

View check run for this annotation

Codecov / codecov/patch

src/array/array_async_readable.rs#L574-L578 

Added lines #L574 - L578 were not covered by tests
.await?;

partial_decoder
.partial_decode_opt(&[array_subset_in_chunk_subset], options)
.partial_decode_opt(&[array_subset_in_chunk_subset], options) // FIXME: Adjust internal decode options

Check warning on line 582 in src/array/array_async_readable.rs

View check run for this annotation

Codecov / codecov/patch

src/array/array_async_readable.rs#L582 

Added line #L582 was not covered by tests
.await?
.remove(0)
};
Expand Down
22 changes: 15 additions & 7 deletions src/array/codec/array_to_bytes/sharding/sharding_codec.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -598,7 +598,8 @@ impl ShardingCodec {
};
if !chunk_representation.fill_value().equals_all(&bytes) {
let chunk_encoded =
self.inner_codecs.encode(bytes, chunk_representation)?;
self.inner_codecs
.encode_opt(bytes, chunk_representation, options)?; // FIXME: Adjust options for inner codec decoding

let chunk_offset = encoded_shard_offset
.fetch_add(chunk_encoded.len(), std::sync::atomic::Ordering::Relaxed);
Expand Down Expand Up @@ -706,7 +707,9 @@ impl ShardingCodec {
if chunk_representation.fill_value().equals_all(&bytes) {
None
} else {
let encoded_chunk = self.inner_codecs.encode(bytes, chunk_representation);
let encoded_chunk =
self.inner_codecs
.encode_opt(bytes, chunk_representation, options); // FIXME: Adjust options for inner chunk encoding
match encoded_chunk {
Ok(encoded_chunk) => Some(Ok((chunk_index, encoded_chunk))),
Err(err) => Some(Err(err)),

Check warning on line 715 in src/array/codec/array_to_bytes/sharding/sharding_codec.rs

View check run for this annotation

Codecov / codecov/patch

src/array/codec/array_to_bytes/sharding/sharding_codec.rs#L715 

Added line #L715 was not covered by tests
Expand Down Expand Up @@ -1436,9 +1439,11 @@ impl ShardingCodec {
let size: usize = size.try_into().unwrap(); // safe
let encoded_chunk_slice = encoded_shard[offset..offset + size].to_vec();
// NOTE: Intentionally using single threaded decode, since parallelisation is in the loop
let decoded = self
.inner_codecs
.decode(encoded_chunk_slice, &chunk_representation)?;
let decoded = self.inner_codecs.decode_opt(
encoded_chunk_slice,
&chunk_representation,
options,
)?; // FIXME: Adjust options for inner chunk decoding
copy_to_subset(&decoded);
};

Expand All @@ -1465,8 +1470,11 @@ impl ShardingCodec {
let offset: usize = offset.try_into().unwrap(); // safe
let size: usize = size.try_into().unwrap(); // safe
let encoded_chunk_slice = encoded_shard[offset..offset + size].to_vec();
self.inner_codecs
.decode(encoded_chunk_slice, &chunk_representation)?
self.inner_codecs.decode_opt(
encoded_chunk_slice,
&chunk_representation,
options,
)? // FIXME: Adjust options for inner chunk encoding
};

// Copy to subset of shard
Expand Down
19 changes: 11 additions & 8 deletions src/array/codec/array_to_bytes/sharding/sharding_partial_decoder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ impl ArrayPartialDecoderTraits for ShardingPartialDecoder<'_> {
fn partial_decode_opt(
&self,
array_subsets: &[ArraySubset],
_options: &PartialDecodeOptions,
options: &PartialDecodeOptions,
) -> Result<Vec<Vec<u8>>, CodecError> {
let Some(shard_index) = &self.shard_index else {
return Ok(array_subsets
Expand Down Expand Up @@ -187,17 +187,18 @@ impl ArrayPartialDecoderTraits for ShardingPartialDecoder<'_> {
fill_value.repeat(array_subset_in_chunk_subset.num_elements_usize())
} else {
// The chunk must be decoded
let partial_decoder = self.inner_codecs.partial_decoder(
let partial_decoder = self.inner_codecs.partial_decoder_opt(
Box::new(ByteIntervalPartialDecoder::new(
&*self.input_handle,
offset,
size,
)),
&chunk_representation,
options, // FIXME: Adjust options for partial decoding
)?;
// NOTE: Intentionally using single threaded decode, since parallelisation is in the loop
partial_decoder
.partial_decode(&[array_subset_in_chunk_subset])?
.partial_decode_opt(&[array_subset_in_chunk_subset], options)? // FIXME: Adjust options for partial decoding
.remove(0)
};

Expand Down Expand Up @@ -429,7 +430,7 @@ impl AsyncArrayPartialDecoderTraits for AsyncShardingPartialDecoder<'_> {
async fn partial_decode_opt(
&self,
array_subsets: &[ArraySubset],
_options: &PartialDecodeOptions,
options: &PartialDecodeOptions,
) -> Result<Vec<Vec<u8>>, CodecError> {
let Some(shard_index) = &self.shard_index else {
return Ok(array_subsets
Expand Down Expand Up @@ -502,13 +503,14 @@ impl AsyncArrayPartialDecoderTraits for AsyncShardingPartialDecoder<'_> {
};
let partial_decoder = self
.inner_codecs
.async_partial_decoder(
.async_partial_decoder_opt(
Box::new(AsyncByteIntervalPartialDecoder::new(
&*self.input_handle,
u64::try_from(*offset).unwrap(),
u64::try_from(*size).unwrap(),
)),
&chunk_representation,
options, // FIXME: Adjust options for partial decoding
)
.await?;
let overlap = unsafe { array_subset.overlap_unchecked(chunk_subset) };
Expand All @@ -521,9 +523,10 @@ impl AsyncArrayPartialDecoderTraits for AsyncShardingPartialDecoder<'_> {
// .await?
// .remove(0);
let decoded_chunk = partial_decoder
.partial_decode(&[ArraySubset::new_with_shape(
chunk_subset.shape().to_vec(),
)])
.partial_decode_opt(
&[ArraySubset::new_with_shape(chunk_subset.shape().to_vec())],
options,
) // FIXME: Adjust options for partial decoding
.await?
.remove(0);
let decoded_chunk = array_subset_in_chunk_subset
Expand Down

0 comments on commit e2eb641

Please sign in to comment.