private static Map<StreamId, DiskRange> getDiskRanges(List<Stream> streams) { ImmutableMap.Builder<StreamId, DiskRange> streamDiskRanges = ImmutableMap.builder(); long stripeOffset = 0; for (Stream stream : streams) { int streamLength = toIntExact(stream.getLength()); // ignore zero byte streams if (streamLength > 0) { streamDiskRanges.put(new StreamId(stream), new DiskRange(stripeOffset, streamLength)); } stripeOffset += streamLength; } return streamDiskRanges.build(); }
private static Map<StreamId, DiskRange> getDiskRanges(List<Stream> streams) { ImmutableMap.Builder<StreamId, DiskRange> streamDiskRanges = ImmutableMap.builder(); long stripeOffset = 0; for (Stream stream : streams) { int streamLength = toIntExact(stream.getLength()); // ignore zero byte streams if (streamLength > 0) { streamDiskRanges.put(new StreamId(stream), new DiskRange(stripeOffset, streamLength)); } stripeOffset += streamLength; } return streamDiskRanges.build(); }
/** * Returns the minimal DiskRange that encloses both this DiskRange * and otherDiskRange. If there was a gap between the ranges the * new range will cover that gap. */ public DiskRange span(DiskRange otherDiskRange) { requireNonNull(otherDiskRange, "otherDiskRange is null"); long start = Math.min(this.offset, otherDiskRange.getOffset()); long end = Math.max(getEnd(), otherDiskRange.getEnd()); return new DiskRange(start, toIntExact(end - start)); }
/** * Returns the minimal DiskRange that encloses both this DiskRange * and otherDiskRange. If there was a gap between the ranges the * new range will cover that gap. */ public DiskRange span(DiskRange otherDiskRange) { requireNonNull(otherDiskRange, "otherDiskRange is null"); long start = Math.min(this.offset, otherDiskRange.getOffset()); long end = Math.max(getEnd(), otherDiskRange.getEnd()); return new DiskRange(start, toIntExact(end - start)); }
public static LinearProbeRangeFinder createTinyStripesRangeFinder(List<StripeInformation> stripes, DataSize maxMergeDistance, DataSize tinyStripeThreshold) { if (stripes.isEmpty()) { return new LinearProbeRangeFinder(ImmutableList.of()); } List<DiskRange> scratchDiskRanges = stripes.stream() .map(stripe -> new DiskRange(stripe.getOffset(), toIntExact(stripe.getTotalLength()))) .collect(Collectors.toList()); List<DiskRange> diskRanges = mergeAdjacentDiskRanges(scratchDiskRanges, maxMergeDistance, tinyStripeThreshold); return new LinearProbeRangeFinder(diskRanges); } }
public static LinearProbeRangeFinder createTinyStripesRangeFinder(List<StripeInformation> stripes, DataSize maxMergeDistance, DataSize tinyStripeThreshold) { if (stripes.size() == 0) { return new LinearProbeRangeFinder(ImmutableList.of()); } List<DiskRange> scratchDiskRanges = stripes.stream() .map(stripe -> new DiskRange(stripe.getOffset(), toIntExact(stripe.getTotalLength()))) .collect(Collectors.toList()); List<DiskRange> diskRanges = mergeAdjacentDiskRanges(scratchDiskRanges, maxMergeDistance, tinyStripeThreshold); return new LinearProbeRangeFinder(diskRanges); } }
@Test public void testMergeAdjacent() { List<DiskRange> diskRanges = mergeAdjacentDiskRanges( ImmutableList.of(new DiskRange(100, 100), new DiskRange(200, 100), new DiskRange(300, 100)), new DataSize(0, BYTE), new DataSize(1, GIGABYTE)); assertEquals(diskRanges, ImmutableList.of(new DiskRange(100, 300))); }
@Test public void testMergeAdjacent() { List<DiskRange> diskRanges = mergeAdjacentDiskRanges( ImmutableList.of(new DiskRange(100, 100), new DiskRange(200, 100), new DiskRange(300, 100)), new DataSize(0, BYTE), new DataSize(1, GIGABYTE)); assertEquals(diskRanges, ImmutableList.of(new DiskRange(100, 300))); }
private static OrcDataSource wrapWithCacheIfTiny(OrcDataSource dataSource, DataSize maxCacheSize) { if (dataSource instanceof CachingOrcDataSource) { return dataSource; } if (dataSource.getSize() > maxCacheSize.toBytes()) { return dataSource; } DiskRange diskRange = new DiskRange(0, toIntExact(dataSource.getSize())); return new CachingOrcDataSource(dataSource, desiredOffset -> diskRange); }
@Override public void readFully(long position, byte[] buffer, int bufferOffset, int bufferLength) throws IOException { readCount++; lastReadRanges = ImmutableList.of(new DiskRange(position, bufferLength)); delegate.readFully(position, buffer, bufferOffset, bufferLength); }
private static OrcDataSource wrapWithCacheIfTiny(OrcDataSource dataSource, DataSize maxCacheSize) { if (dataSource instanceof CachingOrcDataSource) { return dataSource; } if (dataSource.getSize() > maxCacheSize.toBytes()) { return dataSource; } DiskRange diskRange = new DiskRange(0, toIntExact(dataSource.getSize())); return new CachingOrcDataSource(dataSource, desiredOffset -> diskRange); }
@Override public void readFully(long position, byte[] buffer) throws IOException { readCount++; lastReadRanges = ImmutableList.of(new DiskRange(position, buffer.length)); delegate.readFully(position, buffer); }
@Override public void readFully(long position, byte[] buffer, int bufferOffset, int bufferLength) throws IOException { readCount++; lastReadRanges = ImmutableList.of(new DiskRange(position, bufferLength)); delegate.readFully(position, buffer, bufferOffset, bufferLength); }
@Override public void readFully(long position, byte[] buffer) throws IOException { readCount++; lastReadRanges = ImmutableList.of(new DiskRange(position, buffer.length)); delegate.readFully(position, buffer); }
@Test public void testMergeGap() { List<DiskRange> consistent10ByteGap = ImmutableList.of(new DiskRange(100, 90), new DiskRange(200, 90), new DiskRange(300, 90)); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(0, BYTE), new DataSize(1, GIGABYTE)), consistent10ByteGap); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(9, BYTE), new DataSize(1, GIGABYTE)), consistent10ByteGap); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(10, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 290))); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(100, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 290))); List<DiskRange> middle10ByteGap = ImmutableList.of(new DiskRange(100, 80), new DiskRange(200, 90), new DiskRange(300, 80), new DiskRange(400, 90)); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(0, BYTE), new DataSize(1, GIGABYTE)), middle10ByteGap); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(9, BYTE), new DataSize(1, GIGABYTE)), middle10ByteGap); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(10, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 80), new DiskRange(200, 180), new DiskRange(400, 90))); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(100, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 390))); }
@Test public void testMergeMaxSize() { List<DiskRange> consistent10ByteGap = ImmutableList.of(new DiskRange(100, 90), new DiskRange(200, 90), new DiskRange(300, 90)); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(10, BYTE), new DataSize(0, BYTE)), consistent10ByteGap); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(10, BYTE), new DataSize(100, BYTE)), consistent10ByteGap); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(10, BYTE), new DataSize(190, BYTE)), ImmutableList.of(new DiskRange(100, 190), new DiskRange(300, 90))); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(10, BYTE), new DataSize(200, BYTE)), ImmutableList.of(new DiskRange(100, 190), new DiskRange(300, 90))); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(10, BYTE), new DataSize(290, BYTE)), ImmutableList.of(new DiskRange(100, 290))); List<DiskRange> middle10ByteGap = ImmutableList.of(new DiskRange(100, 80), new DiskRange(200, 90), new DiskRange(300, 80), new DiskRange(400, 90)); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(0, BYTE), new DataSize(1, GIGABYTE)), middle10ByteGap); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(9, BYTE), new DataSize(1, GIGABYTE)), middle10ByteGap); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(10, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 80), new DiskRange(200, 180), new DiskRange(400, 90))); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(100, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 390))); } }
@Test public void testMergeGap() { List<DiskRange> consistent10ByteGap = ImmutableList.of(new DiskRange(100, 90), new DiskRange(200, 90), new DiskRange(300, 90)); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(0, BYTE), new DataSize(1, GIGABYTE)), consistent10ByteGap); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(9, BYTE), new DataSize(1, GIGABYTE)), consistent10ByteGap); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(10, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 290))); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(100, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 290))); List<DiskRange> middle10ByteGap = ImmutableList.of(new DiskRange(100, 80), new DiskRange(200, 90), new DiskRange(300, 80), new DiskRange(400, 90)); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(0, BYTE), new DataSize(1, GIGABYTE)), middle10ByteGap); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(9, BYTE), new DataSize(1, GIGABYTE)), middle10ByteGap); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(10, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 80), new DiskRange(200, 180), new DiskRange(400, 90))); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(100, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 390))); }
@Test public void testMergeMaxSize() { List<DiskRange> consistent10ByteGap = ImmutableList.of(new DiskRange(100, 90), new DiskRange(200, 90), new DiskRange(300, 90)); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(10, BYTE), new DataSize(0, BYTE)), consistent10ByteGap); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(10, BYTE), new DataSize(100, BYTE)), consistent10ByteGap); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(10, BYTE), new DataSize(190, BYTE)), ImmutableList.of(new DiskRange(100, 190), new DiskRange(300, 90))); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(10, BYTE), new DataSize(200, BYTE)), ImmutableList.of(new DiskRange(100, 190), new DiskRange(300, 90))); assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(10, BYTE), new DataSize(290, BYTE)), ImmutableList.of(new DiskRange(100, 290))); List<DiskRange> middle10ByteGap = ImmutableList.of(new DiskRange(100, 80), new DiskRange(200, 90), new DiskRange(300, 80), new DiskRange(400, 90)); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(0, BYTE), new DataSize(1, GIGABYTE)), middle10ByteGap); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(9, BYTE), new DataSize(1, GIGABYTE)), middle10ByteGap); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(10, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 80), new DiskRange(200, 180), new DiskRange(400, 90))); assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(100, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 390))); } }