public static boolean[] findPresentStreamsByColumn( List<OrcProto.Stream> streamList, List<OrcProto.Type> types) { boolean[] hasNull = new boolean[types.size()]; for(OrcProto.Stream stream: streamList) { if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.PRESENT)) { hasNull[stream.getColumn()] = true; } } return hasNull; }
public static boolean[] findPresentStreamsByColumn( List<OrcProto.Stream> streamList, List<OrcProto.Type> types) { boolean[] hasNull = new boolean[types.size()]; for(OrcProto.Stream stream: streamList) { if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.PRESENT)) { hasNull[stream.getColumn()] = true; } } return hasNull; }
public static boolean[] findPresentStreamsByColumn( List<OrcProto.Stream> streamList, List<OrcProto.Type> types) { boolean[] hasNull = new boolean[types.size()]; for(OrcProto.Stream stream: streamList) { if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.PRESENT)) { hasNull[stream.getColumn()] = true; } } return hasNull; }
if (stream.hasKind() && range.getOffset() <= offset) { switch (stream.getKind()) { case ROW_INDEX:
OrcProto.Stream.Kind streamKind = stream.getKind(); if (stream.hasKind() && (StreamName.getArea(streamKind) == StreamName.Area.DATA) && (column < includedColumns.length && includedColumns[column])) {
/** * Plan the ranges of the file that we need to read given the list of * columns and row groups. * * @param streamList the list of streams available * @param includedColumns which columns are needed * @param doMergeBuffers * @return the list of disk ranges that will be loaded */ static DiskRangeList planReadPartialDataStreams (List<OrcProto.Stream> streamList, boolean[] includedColumns, boolean doMergeBuffers) { long offset = 0; // figure out which columns have a present stream DiskRangeList.CreateHelper list = new DiskRangeList.CreateHelper(); for (OrcProto.Stream stream : streamList) { long length = stream.getLength(); int column = stream.getColumn(); OrcProto.Stream.Kind streamKind = stream.getKind(); // since stream kind is optional, first check if it exists if (stream.hasKind() && (org.apache.orc.impl.StreamName.getArea(streamKind) == org.apache.orc.impl.StreamName.Area.DATA) && includedColumns[column]) { RecordReaderUtils.addEntireStreamToRanges(offset, length, list, doMergeBuffers); } offset += length; } return list.extract(); }
/** * Plan the ranges of the file that we need to read given the list of * columns and row groups. * * @param streamList the list of streams available * @param includedColumns which columns are needed * @param doMergeBuffers * @return the list of disk ranges that will be loaded */ static DiskRangeList planReadPartialDataStreams (List<OrcProto.Stream> streamList, boolean[] includedColumns, boolean doMergeBuffers) { long offset = 0; // figure out which columns have a present stream DiskRangeList.CreateHelper list = new DiskRangeList.CreateHelper(); for (OrcProto.Stream stream : streamList) { long length = stream.getLength(); int column = stream.getColumn(); OrcProto.Stream.Kind streamKind = stream.getKind(); // since stream kind is optional, first check if it exists if (stream.hasKind() && (org.apache.orc.impl.StreamName.getArea(streamKind) == org.apache.orc.impl.StreamName.Area.DATA) && includedColumns[column]) { RecordReaderUtils.addEntireStreamToRanges(offset, length, list, doMergeBuffers); } offset += length; } return list.extract(); }
void createStreams(List<OrcProto.Stream> streamDescriptions, DiskRangeList ranges, boolean[] includeColumn, CompressionCodec codec, int bufferSize, Map<org.apache.orc.impl.StreamName, InStream> streams) throws IOException { long streamOffset = 0; for (OrcProto.Stream streamDesc : streamDescriptions) { int column = streamDesc.getColumn(); if ((includeColumn != null && !includeColumn[column]) || streamDesc.hasKind() && (org.apache.orc.impl.StreamName.getArea(streamDesc.getKind()) != org.apache.orc.impl.StreamName.Area.DATA)) { streamOffset += streamDesc.getLength(); continue; } List<DiskRange> buffers = RecordReaderUtils.getStreamBuffers( ranges, streamOffset, streamDesc.getLength()); org.apache.orc.impl.StreamName name = new StreamName(column, streamDesc.getKind()); streams.put(name, InStream.create(name.toString(), buffers, streamDesc.getLength(), codec, bufferSize)); streamOffset += streamDesc.getLength(); } }
void createStreams(List<OrcProto.Stream> streamDescriptions, DiskRangeList ranges, boolean[] includeColumn, CompressionCodec codec, int bufferSize, Map<org.apache.orc.impl.StreamName, InStream> streams) throws IOException { long streamOffset = 0; for (OrcProto.Stream streamDesc : streamDescriptions) { int column = streamDesc.getColumn(); if ((includeColumn != null && !includeColumn[column]) || streamDesc.hasKind() && (org.apache.orc.impl.StreamName.getArea(streamDesc.getKind()) != org.apache.orc.impl.StreamName.Area.DATA)) { streamOffset += streamDesc.getLength(); continue; } List<DiskRange> buffers = RecordReaderUtils.getStreamBuffers( ranges, streamOffset, streamDesc.getLength()); org.apache.orc.impl.StreamName name = new StreamName(column, streamDesc.getKind()); streams.put(name, InStream.create(name.toString(), buffers, streamDesc.getLength(), codec, bufferSize)); streamOffset += streamDesc.getLength(); } }
void createStreams(List<OrcProto.Stream> streamDescriptions, DiskRangeList ranges, boolean[] includeColumn, CompressionCodec codec, int bufferSize, Map<StreamName, InStream> streams) throws IOException { long streamOffset = 0; for (OrcProto.Stream streamDesc : streamDescriptions) { int column = streamDesc.getColumn(); if ((includeColumn != null && (column < includeColumn.length && !includeColumn[column])) || streamDesc.hasKind() && (StreamName.getArea(streamDesc.getKind()) != StreamName.Area.DATA)) { streamOffset += streamDesc.getLength(); continue; } List<DiskRange> buffers = RecordReaderUtils.getStreamBuffers( ranges, streamOffset, streamDesc.getLength()); StreamName name = new StreamName(column, streamDesc.getKind()); streams.put(name, InStream.create(name.toString(), buffers, streamDesc.getLength(), codec, bufferSize)); streamOffset += streamDesc.getLength(); } }
public Builder mergeFrom(org.apache.orc.OrcProto.Stream other) { if (other == org.apache.orc.OrcProto.Stream.getDefaultInstance()) return this; if (other.hasKind()) { setKind(other.getKind()); } if (other.hasColumn()) { setColumn(other.getColumn()); } if (other.hasLength()) { setLength(other.getLength()); } this.mergeUnknownFields(other.getUnknownFields()); return this; }