@Override public DiskRange getRangeFor(long desiredOffset) { // Assumption: range are always read in order // Assumption: bytes that are not part of any range are never read for (; index < diskRanges.size(); index++) { DiskRange range = diskRanges.get(index); if (range.getEnd() > desiredOffset) { checkArgument(range.getOffset() <= desiredOffset); return range; } } throw new IllegalArgumentException("Invalid desiredOffset " + desiredOffset); }
@Override public DiskRange getRangeFor(long desiredOffset) { // Assumption: range are always read in order // Assumption: bytes that are not part of any range are never read while (index < diskRanges.size()) { DiskRange range = diskRanges.get(index); if (range.getEnd() > desiredOffset) { checkArgument(range.getOffset() <= desiredOffset); return range; } index++; } throw new IllegalArgumentException("Invalid desiredOffset " + desiredOffset); }
@VisibleForTesting void readCacheAt(long offset) throws IOException { DiskRange newCacheRange = regionFinder.getRangeFor(offset); cachePosition = newCacheRange.getOffset(); cacheLength = newCacheRange.getLength(); if (cache.length < cacheLength) { cache = new byte[cacheLength]; } dataSource.readFully(newCacheRange.getOffset(), cache, 0, cacheLength); }
@VisibleForTesting void readCacheAt(long offset) throws IOException { DiskRange newCacheRange = regionFinder.getRangeFor(offset); cachePosition = newCacheRange.getOffset(); cacheLength = newCacheRange.getLength(); if (cache.length < cacheLength) { cache = new byte[cacheLength]; } dataSource.readFully(newCacheRange.getOffset(), cache, 0, cacheLength); }
/** * Returns the minimal DiskRange that encloses both this DiskRange * and otherDiskRange. If there was a gap between the ranges the * new range will cover that gap. */ public DiskRange span(DiskRange otherDiskRange) { requireNonNull(otherDiskRange, "otherDiskRange is null"); long start = Math.min(this.offset, otherDiskRange.getOffset()); long end = Math.max(getEnd(), otherDiskRange.getEnd()); return new DiskRange(start, toIntExact(end - start)); }
/** * Returns the minimal DiskRange that encloses both this DiskRange * and otherDiskRange. If there was a gap between the ranges the * new range will cover that gap. */ public DiskRange span(DiskRange otherDiskRange) { requireNonNull(otherDiskRange, "otherDiskRange is null"); long start = Math.min(this.offset, otherDiskRange.getOffset()); long end = Math.max(getEnd(), otherDiskRange.getEnd()); return new DiskRange(start, toIntExact(end - start)); }
/** * Get a slice for the disk range from the provided buffers. The buffers ranges do not have * to exactly match {@code diskRange}, but {@code diskRange} must be completely contained within * one of the buffer ranges. */ public static Slice getDiskRangeSlice(DiskRange diskRange, Map<DiskRange, byte[]> buffers) { for (Entry<DiskRange, byte[]> bufferEntry : buffers.entrySet()) { DiskRange bufferRange = bufferEntry.getKey(); byte[] buffer = bufferEntry.getValue(); if (bufferRange.contains(diskRange)) { int offset = toIntExact(diskRange.getOffset() - bufferRange.getOffset()); return Slices.wrappedBuffer(buffer, offset, diskRange.getLength()); } } throw new IllegalStateException("No matching buffer for disk range"); } }
/** * Get a slice for the disk range from the provided buffers. The buffers ranges do not have * to exactly match {@code diskRange}, but {@code diskRange} must be completely contained within * one of the buffer ranges. */ public static Slice getDiskRangeSlice(DiskRange diskRange, Map<DiskRange, byte[]> buffers) { for (Entry<DiskRange, byte[]> bufferEntry : buffers.entrySet()) { DiskRange bufferRange = bufferEntry.getKey(); byte[] buffer = bufferEntry.getValue(); if (bufferRange.contains(diskRange)) { int offset = toIntExact(diskRange.getOffset() - bufferRange.getOffset()); return Slices.wrappedBuffer(buffer, offset, diskRange.getLength()); } } throw new IllegalStateException("No matching buffer for disk range"); } }
@Override public void load(long position, SliceBufferReference bufferReference, int length) { try { readFully(diskRange.getOffset() + position, bufferReference.getBuffer(), 0, length); } catch (IOException e) { throw new UncheckedIOException(e); } }
@Override public void load(long position, SliceBufferReference bufferReference, int length) { try { readFully(diskRange.getOffset() + position, bufferReference.getBuffer(), 0, length); } catch (IOException e) { throw new UncheckedIOException(e); } }
public boolean contains(DiskRange diskRange) { return offset <= diskRange.getOffset() && diskRange.getEnd() <= getEnd(); }
public boolean contains(DiskRange diskRange) { return offset <= diskRange.getOffset() && diskRange.getEnd() <= getEnd(); }
@Override public <K> Map<K, OrcDataSourceInput> readFully(Map<K, DiskRange> diskRanges) throws IOException { ImmutableMap.Builder<K, OrcDataSourceInput> builder = ImmutableMap.builder(); // Assumption here: all disk ranges are in the same region. Therefore, serving them in arbitrary order // will not result in eviction of cache that otherwise could have served any of the DiskRanges provided. for (Map.Entry<K, DiskRange> entry : diskRanges.entrySet()) { DiskRange diskRange = entry.getValue(); byte[] buffer = new byte[diskRange.getLength()]; readFully(diskRange.getOffset(), buffer); builder.put(entry.getKey(), new OrcDataSourceInput(Slices.wrappedBuffer(buffer).getInput(), buffer.length)); } return builder.build(); }
@Override public <K> Map<K, OrcDataSourceInput> readFully(Map<K, DiskRange> diskRanges) throws IOException { ImmutableMap.Builder<K, OrcDataSourceInput> builder = ImmutableMap.builder(); // Assumption here: all disk ranges are in the same region. Therefore, serving them in arbitrary order // will not result in eviction of cache that otherwise could have served any of the DiskRanges provided. for (Map.Entry<K, DiskRange> entry : diskRanges.entrySet()) { DiskRange diskRange = entry.getValue(); byte[] buffer = new byte[diskRange.getLength()]; readFully(diskRange.getOffset(), buffer); builder.put(entry.getKey(), new OrcDataSourceInput(Slices.wrappedBuffer(buffer).getInput(), buffer.length)); } return builder.build(); }
private void load() { if (bufferSlice != null) { return; } try { byte[] buffer = new byte[diskRange.getLength()]; readFully(diskRange.getOffset(), buffer); bufferSlice = Slices.wrappedBuffer(buffer); } catch (IOException e) { throw new UncheckedIOException(e); } } }
private void load() { if (bufferSlice != null) { return; } try { byte[] buffer = new byte[diskRange.getLength()]; readFully(diskRange.getOffset(), buffer); bufferSlice = Slices.wrappedBuffer(buffer); } catch (IOException e) { throw new UncheckedIOException(e); } } }
public Map<StreamId, OrcInputStream> readDiskRanges(long stripeOffset, Map<StreamId, DiskRange> diskRanges, AggregatedMemoryContext systemMemoryUsage) throws IOException { // // Note: this code does not use the Java 8 stream APIs to avoid any extra object allocation // // transform ranges to have an absolute offset in file ImmutableMap.Builder<StreamId, DiskRange> diskRangesBuilder = ImmutableMap.builder(); for (Entry<StreamId, DiskRange> entry : diskRanges.entrySet()) { DiskRange diskRange = entry.getValue(); diskRangesBuilder.put(entry.getKey(), new DiskRange(stripeOffset + diskRange.getOffset(), diskRange.getLength())); } diskRanges = diskRangesBuilder.build(); // read ranges Map<StreamId, OrcDataSourceInput> streamsData = orcDataSource.readFully(diskRanges); // transform streams to OrcInputStream ImmutableMap.Builder<StreamId, OrcInputStream> streamsBuilder = ImmutableMap.builder(); for (Entry<StreamId, OrcDataSourceInput> entry : streamsData.entrySet()) { OrcDataSourceInput sourceInput = entry.getValue(); streamsBuilder.put(entry.getKey(), new OrcInputStream(orcDataSource.getId(), sourceInput.getInput(), decompressor, systemMemoryUsage, sourceInput.getRetainedSizeInBytes())); } return streamsBuilder.build(); }