/** * Helper function to construct exception for parquet schema mismatch. */ private SchemaColumnConvertNotSupportedException constructConvertNotSupportedException( ColumnDescriptor descriptor, WritableColumnVector column) { return new SchemaColumnConvertNotSupportedException( Arrays.toString(descriptor.getPath()), descriptor.getPrimitiveType().getPrimitiveTypeName().toString(), column.dataType().catalogString()); }
/** * Helper function to construct exception for parquet schema mismatch. */ private SchemaColumnConvertNotSupportedException constructConvertNotSupportedException( ColumnDescriptor descriptor, WritableColumnVector column) { return new SchemaColumnConvertNotSupportedException( Arrays.toString(descriptor.getPath()), descriptor.getPrimitiveType().getPrimitiveTypeName().toString(), column.dataType().catalogString()); }
descriptor.getPrimitiveType().getPrimitiveTypeName(); if (isCurrentPageDictionaryEncoded) { case FIXED_LEN_BYTE_ARRAY: readFixedLenByteArrayBatch( rowId, num, column, descriptor.getPrimitiveType().getTypeLength()); break; default:
descriptor.getPrimitiveType().getPrimitiveTypeName(); if (isCurrentPageDictionaryEncoded) { case FIXED_LEN_BYTE_ARRAY: readFixedLenByteArrayBatch( rowId, num, column, descriptor.getPrimitiveType().getTypeLength()); break; default:
private void resetStatistics() { this.statistics = Statistics.createStats(path.getPrimitiveType()); }
private void resetStatistics() { this.statistics = Statistics.createStats(this.path.getPrimitiveType()); }
private void resetStatistics() { this.statistics = Statistics.createStats(path.getPrimitiveType()); }
private ColumnChunkPageWriter(ColumnDescriptor path, BytesCompressor compressor, int initialSlabSize, int maxCapacityHint, ByteBufferAllocator allocator) { this.path = path; this.compressor = compressor; this.buf = new CapacityByteArrayOutputStream(initialSlabSize, maxCapacityHint, allocator); this.totalStatistics = Statistics.createStats(this.path.getPrimitiveType()); }
public IncrementallyUpdatedFilterPredicateBuilderBase(List<PrimitiveColumnIO> leaves) { for (PrimitiveColumnIO leaf : leaves) { ColumnDescriptor descriptor = leaf.getColumnDescriptor(); ColumnPath path = ColumnPath.get(descriptor.getPath()); PrimitiveComparator<?> comparator = descriptor.getPrimitiveType().comparator(); comparatorsByColumn.put(path, comparator); } }
public IncrementallyUpdatedFilterPredicateBuilderBase(List<PrimitiveColumnIO> leaves) { for (PrimitiveColumnIO leaf : leaves) { ColumnDescriptor descriptor = leaf.getColumnDescriptor(); ColumnPath path = ColumnPath.get(descriptor.getPath()); PrimitiveComparator<?> comparator = descriptor.getPrimitiveType().comparator(); comparatorsByColumn.put(path, comparator); } }
private ColumnChunkPageWriter(ColumnDescriptor path, BytesCompressor compressor, ByteBufferAllocator allocator, int columnIndexTruncateLength) { this.path = path; this.compressor = compressor; this.allocator = allocator; this.buf = new ConcatenatingByteArrayCollector(); this.columnIndexBuilder = ColumnIndexBuilder.getBuilder(path.getPrimitiveType(), columnIndexTruncateLength); this.offsetIndexBuilder = OffsetIndexBuilder.getBuilder(); }
/** * start a column inside a block * @param descriptor the column descriptor * @param valueCount the value count in this column * @param compressionCodecName a compression codec name * @throws IOException if there is an error while writing */ public void startColumn(ColumnDescriptor descriptor, long valueCount, CompressionCodecName compressionCodecName) throws IOException { state = state.startColumn(); encodingStatsBuilder.clear(); currentEncodings = new HashSet<Encoding>(); currentChunkPath = ColumnPath.get(descriptor.getPath()); currentChunkType = descriptor.getPrimitiveType(); currentChunkCodec = compressionCodecName; currentChunkValueCount = valueCount; currentChunkFirstDataPage = out.getPos(); compressedLength = 0; uncompressedLength = 0; // The statistics will be copied from the first one added at writeDataPage(s) so we have the correct typed one currentStatistics = null; }
WritableColumnVector column, WritableColumnVector dictionaryIds) { switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { case INT32: if (column.dataType() == DataTypes.IntegerType || "Unsupported type: " + descriptor.getPrimitiveType().getPrimitiveTypeName());
WritableColumnVector column, WritableColumnVector dictionaryIds) { switch (descriptor.getPrimitiveType().getPrimitiveTypeName()) { case INT32: if (column.dataType() == DataTypes.IntegerType || "Unsupported type: " + descriptor.getPrimitiveType().getPrimitiveTypeName());
/** * start a column inside a block * @param descriptor the column descriptor * @param valueCount the value count in this column * @param compressionCodecName a compression codec name * @throws IOException if there is an error while writing */ public void startColumn(ColumnDescriptor descriptor, long valueCount, CompressionCodecName compressionCodecName) throws IOException { state = state.startColumn(); encodingStatsBuilder.clear(); currentEncodings = new HashSet<Encoding>(); currentChunkPath = ColumnPath.get(descriptor.getPath()); currentChunkType = descriptor.getPrimitiveType(); currentChunkCodec = compressionCodecName; currentChunkValueCount = valueCount; currentChunkFirstDataPage = out.getPos(); compressedLength = 0; uncompressedLength = 0; // The statistics will be copied from the first one added at writeDataPage(s) so we have the correct typed one currentStatistics = null; columnIndexBuilder = ColumnIndexBuilder.getBuilder(currentChunkType, columnIndexTruncateLength); offsetIndexBuilder = OffsetIndexBuilder.getBuilder(); firstPageOffset = -1; }
int repetitionLevel = columnReader.getCurrentRepetitionLevel(); ColumnDescriptor column = columnReader.getDescriptor(); PrimitiveType type = column.getPrimitiveType(); if (definitionLevel < column.getMaxDefinitionLevel()) { columnWriter.writeNull(repetitionLevel, definitionLevel);