@Override public long size() { return stream.getLength(); }
private static Map<StreamId, DiskRange> getDiskRanges(List<Stream> streams) { ImmutableMap.Builder<StreamId, DiskRange> streamDiskRanges = ImmutableMap.builder(); long stripeOffset = 0; for (Stream stream : streams) { int streamLength = toIntExact(stream.getLength()); // ignore zero byte streams if (streamLength > 0) { streamDiskRanges.put(new StreamId(stream), new DiskRange(stripeOffset, streamLength)); } stripeOffset += streamLength; } return streamDiskRanges.build(); }
@Override public void writeData(SliceOutput sliceOutput) { long size = writer.applyAsLong(sliceOutput); verify(stream.getLength() == size, "Data stream did not write expected size"); } }
private Map<StreamId, ValueInputStream<?>> createValueStreams(Map<StreamId, Stream> streams, Map<StreamId, OrcInputStream> streamsData, List<ColumnEncoding> columnEncodings) { ImmutableMap.Builder<StreamId, ValueInputStream<?>> valueStreams = ImmutableMap.builder(); for (Entry<StreamId, Stream> entry : streams.entrySet()) { StreamId streamId = entry.getKey(); Stream stream = entry.getValue(); ColumnEncodingKind columnEncoding = columnEncodings.get(stream.getColumn()) .getColumnEncoding(stream.getSequence()) .getColumnEncodingKind(); // skip index and empty streams if (isIndexStream(stream) || stream.getLength() == 0) { continue; } OrcInputStream inputStream = streamsData.get(streamId); OrcTypeKind columnType = types.get(stream.getColumn()).getOrcTypeKind(); valueStreams.put(streamId, ValueStreams.createValueStreams(streamId, inputStream, columnType, columnEncoding, stream.isUseVInts())); } return valueStreams.build(); }
public Optional<StreamDataOutput> getStreamDataOutput(int column) { checkArgument(closed); if (booleanOutputStream == null) { return Optional.empty(); } StreamDataOutput streamDataOutput = booleanOutputStream.getStreamDataOutput(column); // rewrite the DATA stream created by the boolean output stream to a PRESENT stream Stream stream = new Stream(column, PRESENT, toIntExact(streamDataOutput.size()), streamDataOutput.getStream().isUseVInts()); return Optional.of(new StreamDataOutput( sliceOutput -> { streamDataOutput.writeData(sliceOutput); return stream.getLength(); }, stream)); }
private static OrcProto.Stream toStream(Stream stream) { return OrcProto.Stream.newBuilder() .setColumn(stream.getColumn()) .setKind(toStreamKind(stream.getStreamKind())) .setLength(stream.getLength()) .build(); }
private static DwrfProto.Stream toStream(Stream stream) { return DwrfProto.Stream.newBuilder() .setColumn(stream.getColumn()) .setKind(toStreamKind(stream.getStreamKind())) .setLength(stream.getLength()) .setUseVInts(stream.isUseVInts()) .build(); }
assertEquals(stream.getStreamKind(), StreamKind.DATA); assertEquals(stream.getColumn(), 33); assertEquals(stream.getLength(), sliceOutput.size());
assertEquals(stream.getStreamKind(), StreamKind.DATA); assertEquals(stream.getColumn(), 33); assertEquals(stream.getLength(), sliceOutput.size());
@Override public long size() { return stream.getLength(); }
private static Map<StreamId, DiskRange> getDiskRanges(List<Stream> streams) { ImmutableMap.Builder<StreamId, DiskRange> streamDiskRanges = ImmutableMap.builder(); long stripeOffset = 0; for (Stream stream : streams) { int streamLength = toIntExact(stream.getLength()); // ignore zero byte streams if (streamLength > 0) { streamDiskRanges.put(new StreamId(stream), new DiskRange(stripeOffset, streamLength)); } stripeOffset += streamLength; } return streamDiskRanges.build(); }
private static Map<StreamId, DiskRange> getDiskRanges(List<Stream> streams) { ImmutableMap.Builder<StreamId, DiskRange> streamDiskRanges = ImmutableMap.builder(); long stripeOffset = 0; for (Stream stream : streams) { int streamLength = Ints.checkedCast(stream.getLength()); streamDiskRanges.put(new StreamId(stream), new DiskRange(stripeOffset, streamLength)); stripeOffset += streamLength; } return streamDiskRanges.build(); }
@Override public void writeData(SliceOutput sliceOutput) { long size = writer.applyAsLong(sliceOutput); verify(stream.getLength() == size, "Data stream did not write expected size"); } }
private Map<StreamId, ValueStream<?>> createValueStreams(Map<StreamId, Stream> streams, Map<StreamId, OrcInputStream> streamsData, List<ColumnEncoding> columnEncodings) { ImmutableMap.Builder<StreamId, ValueStream<?>> valueStreams = ImmutableMap.builder(); for (Entry<StreamId, Stream> entry : streams.entrySet()) { StreamId streamId = entry.getKey(); Stream stream = entry.getValue(); ColumnEncodingKind columnEncoding = columnEncodings.get(stream.getColumn()).getColumnEncodingKind(); // skip index and empty streams if (isIndexStream(stream) || stream.getLength() == 0) { continue; } OrcInputStream inputStream = streamsData.get(streamId); OrcTypeKind columnType = types.get(stream.getColumn()).getOrcTypeKind(); valueStreams.put(streamId, ValueStreams.createValueStreams(streamId, inputStream, columnType, columnEncoding, stream.isUseVInts())); } return valueStreams.build(); }
private Map<StreamId, ValueInputStream<?>> createValueStreams(Map<StreamId, Stream> streams, Map<StreamId, OrcInputStream> streamsData, List<ColumnEncoding> columnEncodings) { ImmutableMap.Builder<StreamId, ValueInputStream<?>> valueStreams = ImmutableMap.builder(); for (Entry<StreamId, Stream> entry : streams.entrySet()) { StreamId streamId = entry.getKey(); Stream stream = entry.getValue(); ColumnEncodingKind columnEncoding = columnEncodings.get(stream.getColumn()) .getColumnEncoding(stream.getSequence()) .getColumnEncodingKind(); // skip index and empty streams if (isIndexStream(stream) || stream.getLength() == 0) { continue; } OrcInputStream inputStream = streamsData.get(streamId); OrcTypeKind columnType = types.get(stream.getColumn()).getOrcTypeKind(); valueStreams.put(streamId, ValueStreams.createValueStreams(streamId, inputStream, columnType, columnEncoding, stream.isUseVInts())); } return valueStreams.build(); }
public Optional<StreamDataOutput> getStreamDataOutput(int column) { checkArgument(closed); if (booleanOutputStream == null) { return Optional.empty(); } StreamDataOutput streamDataOutput = booleanOutputStream.getStreamDataOutput(column); // rewrite the DATA stream created by the boolean output stream to a PRESENT stream Stream stream = new Stream(column, PRESENT, toIntExact(streamDataOutput.size()), streamDataOutput.getStream().isUseVInts()); return Optional.of(new StreamDataOutput( sliceOutput -> { streamDataOutput.writeData(sliceOutput); return stream.getLength(); }, stream)); }
private static OrcProto.Stream toStream(Stream stream) { return OrcProto.Stream.newBuilder() .setColumn(stream.getColumn()) .setKind(toStreamKind(stream.getStreamKind())) .setLength(stream.getLength()) .build(); }
private static DwrfProto.Stream toStream(Stream stream) { return DwrfProto.Stream.newBuilder() .setColumn(stream.getColumn()) .setKind(toStreamKind(stream.getStreamKind())) .setLength(stream.getLength()) .setUseVInts(stream.isUseVInts()) .build(); }
assertEquals(stream.getStreamKind(), StreamKind.DATA); assertEquals(stream.getColumn(), 33); assertEquals(stream.getLength(), sliceOutput.size());