@Override public <K> Map<K, OrcDataSourceInput> readFully(Map<K, DiskRange> diskRanges) throws IOException { readCount += diskRanges.size(); lastReadRanges = ImmutableList.copyOf(diskRanges.values()); return delegate.readFully(diskRanges); } }
@Override public <K> Map<K, OrcDataSourceInput> readFully(Map<K, DiskRange> diskRanges) throws IOException { readCount += diskRanges.size(); lastReadRanges = ImmutableList.copyOf(diskRanges.values()); return delegate.readFully(diskRanges); } }
public Map<StreamId, OrcInputStream> readDiskRanges(long stripeOffset, Map<StreamId, DiskRange> diskRanges, AggregatedMemoryContext systemMemoryUsage) throws IOException { // // Note: this code does not use the Java 8 stream APIs to avoid any extra object allocation // // transform ranges to have an absolute offset in file ImmutableMap.Builder<StreamId, DiskRange> diskRangesBuilder = ImmutableMap.builder(); for (Entry<StreamId, DiskRange> entry : diskRanges.entrySet()) { DiskRange diskRange = entry.getValue(); diskRangesBuilder.put(entry.getKey(), new DiskRange(stripeOffset + diskRange.getOffset(), diskRange.getLength())); } diskRanges = diskRangesBuilder.build(); // read ranges Map<StreamId, OrcDataSourceInput> streamsData = orcDataSource.readFully(diskRanges); // transform streams to OrcInputStream ImmutableMap.Builder<StreamId, OrcInputStream> streamsBuilder = ImmutableMap.builder(); for (Entry<StreamId, OrcDataSourceInput> entry : streamsData.entrySet()) { OrcDataSourceInput sourceInput = entry.getValue(); streamsBuilder.put(entry.getKey(), new OrcInputStream(orcDataSource.getId(), sourceInput.getInput(), decompressor, systemMemoryUsage, sourceInput.getRetainedSizeInBytes())); } return streamsBuilder.build(); }
public Map<StreamId, OrcInputStream> readDiskRanges(long stripeOffset, Map<StreamId, DiskRange> diskRanges, AggregatedMemoryContext systemMemoryUsage) throws IOException { // // Note: this code does not use the Java 8 stream APIs to avoid any extra object allocation // // transform ranges to have an absolute offset in file ImmutableMap.Builder<StreamId, DiskRange> diskRangesBuilder = ImmutableMap.builder(); for (Entry<StreamId, DiskRange> entry : diskRanges.entrySet()) { DiskRange diskRange = entry.getValue(); diskRangesBuilder.put(entry.getKey(), new DiskRange(stripeOffset + diskRange.getOffset(), diskRange.getLength())); } diskRanges = diskRangesBuilder.build(); // read ranges Map<StreamId, OrcDataSourceInput> streamsData = orcDataSource.readFully(diskRanges); // transform streams to OrcInputStream ImmutableMap.Builder<StreamId, OrcInputStream> streamsBuilder = ImmutableMap.builder(); for (Entry<StreamId, OrcDataSourceInput> entry : streamsData.entrySet()) { OrcDataSourceInput sourceInput = entry.getValue(); streamsBuilder.put(entry.getKey(), new OrcInputStream(orcDataSource.getId(), sourceInput.getInput(), decompressor, systemMemoryUsage, sourceInput.getRetainedSizeInBytes())); } return streamsBuilder.build(); }
@Override public void readFully(long position, byte[] buffer, int bufferOffset, int bufferLength) throws IOException { readCount++; lastReadRanges = ImmutableList.of(new DiskRange(position, bufferLength)); delegate.readFully(position, buffer, bufferOffset, bufferLength); }
/** * Does the file start with the ORC magic bytes? */ private static boolean isValidHeaderMagic(OrcDataSource source) throws IOException { byte[] headerMagic = new byte[MAGIC.length()]; source.readFully(0, headerMagic); return MAGIC.equals(Slices.wrappedBuffer(headerMagic)); }
@Override public void readFully(long position, byte[] buffer) throws IOException { readCount++; lastReadRanges = ImmutableList.of(new DiskRange(position, buffer.length)); delegate.readFully(position, buffer); }
@Override public void readFully(long position, byte[] buffer, int bufferOffset, int bufferLength) throws IOException { readCount++; lastReadRanges = ImmutableList.of(new DiskRange(position, bufferLength)); delegate.readFully(position, buffer, bufferOffset, bufferLength); }
@VisibleForTesting void readCacheAt(long offset) throws IOException { DiskRange newCacheRange = regionFinder.getRangeFor(offset); cachePosition = newCacheRange.getOffset(); cacheLength = newCacheRange.getLength(); if (cache.length < cacheLength) { cache = new byte[cacheLength]; } dataSource.readFully(newCacheRange.getOffset(), cache, 0, cacheLength); }
/** * Does the file start with the ORC magic bytes? */ private static boolean isValidHeaderMagic(OrcDataSource source) throws IOException { byte[] headerMagic = new byte[MAGIC.length()]; source.readFully(0, headerMagic); return MAGIC.equals(Slices.wrappedBuffer(headerMagic)); }
@Override public void readFully(long position, byte[] buffer) throws IOException { readCount++; lastReadRanges = ImmutableList.of(new DiskRange(position, buffer.length)); delegate.readFully(position, buffer); }
@VisibleForTesting void readCacheAt(long offset) throws IOException { DiskRange newCacheRange = regionFinder.getRangeFor(offset); cachePosition = newCacheRange.getOffset(); cacheLength = newCacheRange.getLength(); if (cache.length < cacheLength) { cache = new byte[cacheLength]; } dataSource.readFully(newCacheRange.getOffset(), cache, 0, cacheLength); }
public StripeFooter readStripeFooter(StripeInformation stripe, AggregatedMemoryContext systemMemoryUsage) throws IOException { long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(); int tailLength = toIntExact(stripe.getFooterLength()); // read the footer byte[] tailBuffer = new byte[tailLength]; orcDataSource.readFully(offset, tailBuffer); try (InputStream inputStream = new OrcInputStream(orcDataSource.getId(), Slices.wrappedBuffer(tailBuffer).getInput(), decompressor, systemMemoryUsage, tailLength)) { return metadataReader.readStripeFooter(types, inputStream); } }
public StripeFooter readStripeFooter(StripeInformation stripe, AggregatedMemoryContext systemMemoryUsage) throws IOException { long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(); int tailLength = toIntExact(stripe.getFooterLength()); // read the footer byte[] tailBuffer = new byte[tailLength]; orcDataSource.readFully(offset, tailBuffer); try (InputStream inputStream = new OrcInputStream(orcDataSource.getId(), Slices.wrappedBuffer(tailBuffer).getInput(), decompressor, systemMemoryUsage, tailLength)) { return metadataReader.readStripeFooter(types, inputStream); } }
orcDataSource.readFully(stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(), tailBuffer); try (InputStream inputStream = new OrcInputStream(orcDataSource.getId(), Slices.wrappedBuffer(tailBuffer).getInput(), Optional.empty(), newSimpleAggregatedMemoryContext(), tailBuffer.length)) { StripeFooter stripeFooter = ORC.createMetadataReader().readStripeFooter(footer.getTypes(), inputStream);
orcDataSource.readFully(stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(), tailBuffer); try (InputStream inputStream = new OrcInputStream(orcDataSource.getId(), Slices.wrappedBuffer(tailBuffer).getInput(), Optional.empty(), newSimpleAggregatedMemoryContext(), tailBuffer.length)) { StripeFooter stripeFooter = ORC.createMetadataReader().readStripeFooter(footer.getTypes(), inputStream);