@Override public FixedLengthSliceInput get() { return new ChunkedSliceInput(new ChunkedSliceLoader(diskRange), bufferSize); } }
this.input = new ChunkedSliceInput(new DataSourceSliceLoader(dataSource), toIntExact(bufferSize.toBytes()));
private <K> Map<K, FixedLengthSliceInput> readLargeDiskRanges(Map<K, DiskRange> diskRanges) throws IOException { if (diskRanges.isEmpty()) { return ImmutableMap.of(); } ImmutableMap.Builder<K, FixedLengthSliceInput> slices = ImmutableMap.builder(); for (Entry<K, DiskRange> entry : diskRanges.entrySet()) { ChunkedSliceInput sliceInput = new ChunkedSliceInput(new HdfsSliceLoader(entry.getValue()), Ints.checkedCast(streamBufferSize.toBytes())); slices.put(entry.getKey(), sliceInput); } return slices.build(); }
@Override public FixedLengthSliceInput get() { return new ChunkedSliceInput(new ChunkedSliceLoader(diskRange), bufferSize); } }
@Override public FixedLengthSliceInput get() { return new ChunkedSliceInput(new ChunkedSliceLoader(diskRange), bufferSize); } }
@Override public FixedLengthSliceInput get() { return new ChunkedSliceInput(new ChunkedSliceLoader(diskRange), bufferSize); } }
@Override protected SliceInput createSliceInput(Slice slice) { return new ChunkedSliceInput(new SliceSliceLoader(slice), BUFFER_SIZE); }
@Override protected SliceInput createSliceInput(Slice slice) { return new ChunkedSliceInput(new SliceSliceLoader(slice), BUFFER_SIZE); }
@Test public void testSetPosition() { // Create a ChunkedSliceInput with 160 bytes, and set bufferSize to 130. // This test read bytes sequentially, but calls setPosition every time. // Only 2 reads should be issued to the underlying SliceLoader. One when // trying to read position 0, and one when trying to read position 130. int length = 160; int bufferSize = 130; Slice slice = Slices.utf8Slice(Strings.repeat("0", length)); SliceSliceLoader loader = new SliceSliceLoader(slice); ChunkedSliceInput chunkedSliceInput = new ChunkedSliceInput(loader, bufferSize); ArrayList<Integer> actual = new ArrayList<>(); for (int i = 0; i < length; i++) { chunkedSliceInput.setPosition(i); chunkedSliceInput.readByte(); int count = loader.getCount(); actual.add(count); } List<Integer> expected = IntStream.range(0, length) .map(i -> i < bufferSize ? 1 : 2) .boxed() .collect(Collectors.toList()); assertEquals(actual, expected); }
@Test public void testSetPosition() { // Create a ChunkedSliceInput with 160 bytes, and set bufferSize to 130. // This test read bytes sequentially, but calls setPosition every time. // Only 2 reads should be issued to the underlying SliceLoader. One when // trying to read position 0, and one when trying to read position 130. int length = 160; int bufferSize = 130; Slice slice = Slices.utf8Slice(Strings.repeat("0", length)); SliceSliceLoader loader = new SliceSliceLoader(slice); ChunkedSliceInput chunkedSliceInput = new ChunkedSliceInput(loader, bufferSize); ArrayList<Integer> actual = new ArrayList<>(); for (int i = 0; i < length; i++) { chunkedSliceInput.setPosition(i); chunkedSliceInput.readByte(); int count = loader.getCount(); actual.add(count); } List<Integer> expected = IntStream.range(0, length) .map(i -> i < bufferSize ? 1 : 2) .boxed() .collect(Collectors.toList()); assertEquals(actual, expected); }
@Test public void testRetainedSize() { int length = 1024; int bufferSize = 128; Slice slice = Slices.utf8Slice(Strings.repeat("0", length)); SliceSliceLoader loader = new SliceSliceLoader(slice); ChunkedSliceInput chunkedSliceInput = new ChunkedSliceInput(loader, bufferSize); for (int i = 0; i < length; i++) { chunkedSliceInput.setPosition(i); chunkedSliceInput.readByte(); assertRetainedSize(chunkedSliceInput); long retainedSize = chunkedSliceInput.getRetainedSize(); assertTrue(retainedSize > bufferSize && retainedSize < length); } }
@Test public void testRetainedSize() { int length = 1024; int bufferSize = 128; Slice slice = Slices.utf8Slice(Strings.repeat("0", length)); SliceSliceLoader loader = new SliceSliceLoader(slice); ChunkedSliceInput chunkedSliceInput = new ChunkedSliceInput(loader, bufferSize); for (int i = 0; i < length; i++) { chunkedSliceInput.setPosition(i); chunkedSliceInput.readByte(); assertRetainedSize(chunkedSliceInput); long retainedSize = chunkedSliceInput.getRetainedSize(); assertTrue(retainedSize > bufferSize && retainedSize < length); } }
this.input = new ChunkedSliceInput(new DataSourceSliceLoader(dataSource), toIntExact(bufferSize.toBytes()));
this.input = new ChunkedSliceInput(new DataSourceSliceLoader(dataSource), toIntExact(bufferSize.toBytes()));