public static Optional<RichColumnDescriptor> getDescriptor(List<PrimitiveColumnIO> columns, List<String> path) { checkArgument(path.size() >= 1, "Parquet nested path should have at least one component"); int index = getPathIndex(columns, path); if (index == -1) { return Optional.empty(); } PrimitiveColumnIO columnIO = columns.get(index); return Optional.of(new RichColumnDescriptor(columnIO.getColumnDescriptor(), columnIO.getType().asPrimitiveType())); }
public static Optional<RichColumnDescriptor> getDescriptor(List<PrimitiveColumnIO> columns, List<String> path) { checkArgument(path.size() >= 1, "Parquet nested path should have at least one component"); int index = getPathIndex(columns, path); if (index == -1) { return Optional.empty(); } PrimitiveColumnIO columnIO = columns.get(index); return Optional.of(new RichColumnDescriptor(columnIO.getColumnDescriptor(), columnIO.getType().asPrimitiveType())); }
public IncrementallyUpdatedFilterPredicateBuilderBase(List<PrimitiveColumnIO> leaves) { for (PrimitiveColumnIO leaf : leaves) { ColumnDescriptor descriptor = leaf.getColumnDescriptor(); ColumnPath path = ColumnPath.get(descriptor.getPath()); PrimitiveComparator<?> comparator = descriptor.getPrimitiveType().comparator(); comparatorsByColumn.put(path, comparator); } }
public IncrementallyUpdatedFilterPredicateBuilderBase(List<PrimitiveColumnIO> leaves) { for (PrimitiveColumnIO leaf : leaves) { ColumnDescriptor descriptor = leaf.getColumnDescriptor(); ColumnPath path = ColumnPath.get(descriptor.getPath()); PrimitiveComparator<?> comparator = descriptor.getPrimitiveType().comparator(); comparatorsByColumn.put(path, comparator); } }
public MessageColumnIORecordConsumer(ColumnWriteStore columns) { this.columns = columns; int maxDepth = 0; this.columnWriter = new ColumnWriter[MessageColumnIO.this.getLeaves().size()]; for (PrimitiveColumnIO primitiveColumnIO : MessageColumnIO.this.getLeaves()) { ColumnWriter w = columns.getColumnWriter(primitiveColumnIO.getColumnDescriptor()); maxDepth = Math.max(maxDepth, primitiveColumnIO.getFieldPath().length); columnWriter[primitiveColumnIO.getId()] = w; buildGroupToLeafWriterMap(primitiveColumnIO, w); } fieldsWritten = new FieldsMarker[maxDepth]; for (int i = 0; i < maxDepth; i++) { fieldsWritten[i] = new FieldsMarker(); } r = new int[maxDepth]; }
public MessageColumnIORecordConsumer(ColumnWriteStore columns) { this.columns = columns; int maxDepth = 0; this.columnWriter = new ColumnWriter[MessageColumnIO.this.getLeaves().size()]; for (PrimitiveColumnIO primitiveColumnIO : MessageColumnIO.this.getLeaves()) { ColumnWriter w = columns.getColumnWriter(primitiveColumnIO.getColumnDescriptor()); maxDepth = Math.max(maxDepth, primitiveColumnIO.getFieldPath().length); columnWriter[primitiveColumnIO.getId()] = w; buildGroupToLeafWriterMap(primitiveColumnIO, w); } fieldsWritten = new FieldsMarker[maxDepth]; for (int i = 0; i < maxDepth; i++) { fieldsWritten[i] = new FieldsMarker(); } r = new int[maxDepth]; }
private void initializeColumnReaders() { for (PrimitiveColumnIO columnIO : columns) { RichColumnDescriptor column = new RichColumnDescriptor(columnIO.getColumnDescriptor(), columnIO.getType().asPrimitiveType()); columnReaders[columnIO.getId()] = PrimitiveColumnReader.createReader(column); } }
private void initializeColumnReaders() { for (PrimitiveColumnIO columnIO : columns) { RichColumnDescriptor column = new RichColumnDescriptor(columnIO.getColumnDescriptor(), columnIO.getType().asPrimitiveType()); columnReaders[columnIO.getId()] = PrimitiveColumnReader.createReader(column); } }
@Override public Converter getConverter(int fieldIndex) { // get the real converter from the delegate Converter delegateConverter = checkNotNull(delegate.getConverter(fieldIndex), "delegate converter"); // determine the indexFieldPath for the converter proxy we're about to make, which is // this converter's path + the requested fieldIndex List<Integer> newIndexFieldPath = new ArrayList<Integer>(indexFieldPath.size() + 1); newIndexFieldPath.addAll(indexFieldPath); newIndexFieldPath.add(fieldIndex); if (delegateConverter.isPrimitive()) { PrimitiveColumnIO columnIO = getColumnIO(newIndexFieldPath); ColumnPath columnPath = ColumnPath.get(columnIO.getColumnDescriptor().getPath()); ValueInspector[] valueInspectors = getValueInspectors(columnPath); return new FilteringPrimitiveConverter(delegateConverter.asPrimitiveConverter(), valueInspectors); } else { return new FilteringGroupConverter(delegateConverter.asGroupConverter(), newIndexFieldPath, valueInspectorsByColumn, columnIOsByIndexFieldPath); } }
@Override public Converter getConverter(int fieldIndex) { // get the real converter from the delegate Converter delegateConverter = checkNotNull(delegate.getConverter(fieldIndex), "delegate converter"); // determine the indexFieldPath for the converter proxy we're about to make, which is // this converter's path + the requested fieldIndex List<Integer> newIndexFieldPath = new ArrayList<Integer>(indexFieldPath.size() + 1); newIndexFieldPath.addAll(indexFieldPath); newIndexFieldPath.add(fieldIndex); if (delegateConverter.isPrimitive()) { PrimitiveColumnIO columnIO = getColumnIO(newIndexFieldPath); ColumnPath columnPath = ColumnPath.get(columnIO.getColumnDescriptor().getPath()); ValueInspector[] valueInspectors = getValueInspectors(columnPath); return new FilteringPrimitiveConverter(delegateConverter.asPrimitiveConverter(), valueInspectors); } else { return new FilteringGroupConverter(delegateConverter.asGroupConverter(), newIndexFieldPath, valueInspectorsByColumn, columnIOsByIndexFieldPath); } }
RichColumnDescriptor column = new RichColumnDescriptor(primitiveColumnIO.getColumnDescriptor(), columnIO.getType().asPrimitiveType()); return Optional.of(new PrimitiveField(type, repetitionLevel, definitionLevel, required, column, primitiveColumnIO.getId()));
columnReaders[i] = columnStore.getColumnReader(leafColumnIO.getColumnDescriptor()); int maxRepetitionLevel = leafColumnIO.getRepetitionLevel(); nextColumnIdxForRepLevel[i] = new int[maxRepetitionLevel+1];
columnReaders[i] = columnStore.getColumnReader(leafColumnIO.getColumnDescriptor()); int maxRepetitionLevel = leafColumnIO.getRepetitionLevel(); nextColumnIdxForRepLevel[i] = new int[maxRepetitionLevel+1];