private void readIntBatch(int rowId, int num, WritableColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. // TODO: implement remaining type conversions if (column.dataType() == DataTypes.IntegerType || column.dataType() == DataTypes.DateType || DecimalType.is32BitDecimalType(column.dataType())) { defColumn.readIntegers( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else if (column.dataType() == DataTypes.ByteType) { defColumn.readBytes( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else if (column.dataType() == DataTypes.ShortType) { defColumn.readShorts( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else { throw constructConvertNotSupportedException(descriptor, column); } }
private void readIntBatch(int rowId, int num, WritableColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. // TODO: implement remaining type conversions if (column.dataType() == DataTypes.IntegerType || column.dataType() == DataTypes.DateType || DecimalType.is32BitDecimalType(column.dataType())) { defColumn.readIntegers( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else if (column.dataType() == DataTypes.ByteType) { defColumn.readBytes( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else if (column.dataType() == DataTypes.ShortType) { defColumn.readShorts( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else { throw constructConvertNotSupportedException(descriptor, column); } }
public final int appendNotNulls(int count) { assert (!(dataType() instanceof StructType)); reserve(elementsAppended + count); int result = elementsAppended; putNotNulls(elementsAppended, count); elementsAppended += count; return result; }
/** * For all the read*Batch functions, reads `num` values from this columnReader into column. It * is guaranteed that num is smaller than the number of values left in the current page. */ private void readBooleanBatch(int rowId, int num, WritableColumnVector column) throws IOException { if (column.dataType() != DataTypes.BooleanType) { throw constructConvertNotSupportedException(descriptor, column); } defColumn.readBooleans( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); }
/** * Append APIs. These APIs all behave similarly and will append data to the current vector. It * is not valid to mix the put and append APIs. The append APIs are slower and should only be * used if the sizes are not known up front. * In all these cases, the return value is the rowId for the first appended element. */ public final int appendNull() { assert (!(dataType() instanceof StructType)); // Use appendStruct() reserve(elementsAppended + 1); putNull(elementsAppended); return elementsAppended++; }
/** * For all the read*Batch functions, reads `num` values from this columnReader into column. It * is guaranteed that num is smaller than the number of values left in the current page. */ private void readBooleanBatch(int rowId, int num, WritableColumnVector column) throws IOException { if (column.dataType() != DataTypes.BooleanType) { throw constructConvertNotSupportedException(descriptor, column); } defColumn.readBooleans( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); }
private void readFloatBatch(int rowId, int num, WritableColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. // TODO: support implicit cast to double? if (column.dataType() == DataTypes.FloatType) { defColumn.readFloats( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else { throw constructConvertNotSupportedException(descriptor, column); } }
private void readDoubleBatch(int rowId, int num, WritableColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. // TODO: implement remaining type conversions if (column.dataType() == DataTypes.DoubleType) { defColumn.readDoubles( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else { throw constructConvertNotSupportedException(descriptor, column); } }
public final int appendNotNulls(int count) { assert (!(dataType() instanceof StructType)); reserve(elementsAppended + count); int result = elementsAppended; putNotNulls(elementsAppended, count); elementsAppended += count; return result; }
public final int appendNulls(int count) { assert (!(dataType() instanceof StructType)); reserve(elementsAppended + count); int result = elementsAppended; putNulls(elementsAppended, count); elementsAppended += count; return result; }
private void readDoubleBatch(int rowId, int num, WritableColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. // TODO: implement remaining type conversions if (column.dataType() == DataTypes.DoubleType) { defColumn.readDoubles( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else { throw constructConvertNotSupportedException(descriptor, column); } }
public final int appendNulls(int count) { assert (!(dataType() instanceof StructType)); reserve(elementsAppended + count); int result = elementsAppended; putNulls(elementsAppended, count); elementsAppended += count; return result; }
/** * Append APIs. These APIs all behave similarly and will append data to the current vector. It * is not valid to mix the put and append APIs. The append APIs are slower and should only be * used if the sizes are not known up front. * In all these cases, the return value is the rowId for the first appended element. */ public final int appendNull() { assert (!(dataType() instanceof StructType)); // Use appendStruct() reserve(elementsAppended + 1); putNull(elementsAppended); return elementsAppended++; }
private void readFloatBatch(int rowId, int num, WritableColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. // TODO: support implicit cast to double? if (column.dataType() == DataTypes.FloatType) { defColumn.readFloats( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else { throw constructConvertNotSupportedException(descriptor, column); } }
/** * Helper function to construct exception for parquet schema mismatch. */ private SchemaColumnConvertNotSupportedException constructConvertNotSupportedException( ColumnDescriptor descriptor, WritableColumnVector column) { return new SchemaColumnConvertNotSupportedException( Arrays.toString(descriptor.getPath()), descriptor.getPrimitiveType().getPrimitiveTypeName().toString(), column.dataType().catalogString()); }
/** * Helper function to construct exception for parquet schema mismatch. */ private SchemaColumnConvertNotSupportedException constructConvertNotSupportedException( ColumnDescriptor descriptor, WritableColumnVector column) { return new SchemaColumnConvertNotSupportedException( Arrays.toString(descriptor.getPath()), descriptor.getPrimitiveType().getPrimitiveTypeName().toString(), column.dataType().catalogString()); }
private void readLongBatch(int rowId, int num, WritableColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. if (column.dataType() == DataTypes.LongType || DecimalType.is64BitDecimalType(column.dataType()) || originalType == OriginalType.TIMESTAMP_MICROS) { defColumn.readLongs( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else if (originalType == OriginalType.TIMESTAMP_MILLIS) { for (int i = 0; i < num; i++) { if (defColumn.readInteger() == maxDefLevel) { column.putLong(rowId + i, DateTimeUtils.fromMillis(dataColumn.readLong())); } else { column.putNull(rowId + i); } } } else { throw constructConvertNotSupportedException(descriptor, column); } }
private void readLongBatch(int rowId, int num, WritableColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. if (column.dataType() == DataTypes.LongType || DecimalType.is64BitDecimalType(column.dataType()) || originalType == OriginalType.TIMESTAMP_MICROS) { defColumn.readLongs( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else if (originalType == OriginalType.TIMESTAMP_MILLIS) { for (int i = 0; i < num; i++) { if (defColumn.readInteger() == maxDefLevel) { column.putLong(rowId + i, DateTimeUtils.fromMillis(dataColumn.readLong())); } else { column.putNull(rowId + i); } } } else { throw constructConvertNotSupportedException(descriptor, column); } }
if (column.dataType() == DataTypes.StringType || column.dataType() == DataTypes.BinaryType || DecimalType.isByteArrayDecimalType(column.dataType())) { defColumn.readBinarys(num, column, rowId, maxDefLevel, data); } else if (column.dataType() == DataTypes.TimestampType) { if (!shouldConvertTimestamps()) { for (int i = 0; i < num; i++) {
if (column.dataType() == DataTypes.StringType || column.dataType() == DataTypes.BinaryType || DecimalType.isByteArrayDecimalType(column.dataType())) { defColumn.readBinarys(num, column, rowId, maxDefLevel, data); } else if (column.dataType() == DataTypes.TimestampType) { if (!shouldConvertTimestamps()) { for (int i = 0; i < num; i++) {