private void readFloatBatch(int rowId, int num, ColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. // TODO: support implicit cast to double? if (column.dataType() == DataTypes.FloatType) { defColumn.readFloats( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else { throw new UnsupportedOperationException("Unsupported conversion to: " + column.dataType()); } }
private void readDoubleBatch(int rowId, int num, ColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. // TODO: implement remaining type conversions if (column.dataType() == DataTypes.DoubleType) { defColumn.readDoubles( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else { throw new UnsupportedOperationException("Unimplemented type: " + column.dataType()); } }
/** * For all the read*Batch functions, reads `num` values from this columnReader into column. It * is guaranteed that num is smaller than the number of values left in the current page. */ private void readBooleanBatch(int rowId, int num, ColumnVector column) throws IOException { assert(column.dataType() == DataTypes.BooleanType); defColumn.readBooleans( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); }
private void readIntBatch(int rowId, int num, ColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. // TODO: implement remaining type conversions if (column.dataType() == DataTypes.IntegerType || column.dataType() == DataTypes.DateType || DecimalType.is32BitDecimalType(column.dataType())) { defColumn.readIntegers( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else if (column.dataType() == DataTypes.ByteType) { defColumn.readBytes( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else if (column.dataType() == DataTypes.ShortType) { defColumn.readShorts( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else { throw new UnsupportedOperationException("Unimplemented type: " + column.dataType()); } }
public final int appendNulls(int count) { assert (!(dataType() instanceof StructType)); reserve(elementsAppended + count); int result = elementsAppended; putNulls(elementsAppended, count); elementsAppended += count; return result; }
public final int appendNotNulls(int count) { assert (!(dataType() instanceof StructType)); reserve(elementsAppended + count); int result = elementsAppended; putNotNulls(elementsAppended, count); elementsAppended += count; return result; }
/** * Append APIs. These APIs all behave similarly and will append data to the current vector. It * is not valid to mix the put and append APIs. The append APIs are slower and should only be * used if the sizes are not known up front. * In all these cases, the return value is the rowId for the first appended element. */ public final int appendNull() { assert (!(dataType() instanceof StructType)); // Use appendStruct() reserve(elementsAppended + 1); putNull(elementsAppended); return elementsAppended++; }
/** * Returns the array data as the java primitive array. * For example, an array of IntegerType will return an int[]. * Throws exceptions for unhandled schemas. */ public static Object toPrimitiveJavaArray(ColumnVector.Array array) { DataType dt = array.data.dataType(); if (dt instanceof IntegerType) { int[] result = new int[array.length]; ColumnVector data = array.data; for (int i = 0; i < result.length; i++) { if (data.isNullAt(array.offset + i)) { throw new RuntimeException("Cannot handle NULL values."); } result[i] = data.getInt(array.offset + i); } return result; } else { throw new UnsupportedOperationException(); } }
private void readLongBatch(int rowId, int num, ColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. if (column.dataType() == DataTypes.LongType || DecimalType.is64BitDecimalType(column.dataType())) { defColumn.readLongs( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else if (column.dataType() == DataTypes.TimestampType) { for (int i = 0; i < num; i++) { if (defColumn.readInteger() == maxDefLevel) { column.putLong(rowId + i, DateTimeUtils.fromMillis(dataColumn.readLong())); } else { column.putNull(rowId + i); } } } else { throw new UnsupportedOperationException("Unsupported conversion to: " + column.dataType()); } }
private void readBinaryBatch(int rowId, int num, ColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. // TODO: implement remaining type conversions VectorizedValuesReader data = (VectorizedValuesReader) dataColumn; if (column.isArray()) { defColumn.readBinarys(num, column, rowId, maxDefLevel, data); } else if (column.dataType() == DataTypes.TimestampType) { for (int i = 0; i < num; i++) { if (defColumn.readInteger() == maxDefLevel) { column.putLong(rowId + i, // Read 12 bytes for INT96 ParquetRowConverter.binaryToSQLTimestamp(data.readBinary(12))); } else { column.putNull(rowId + i); } } } else { throw new UnsupportedOperationException("Unimplemented type: " + column.dataType()); } }
if (DecimalType.is32BitDecimalType(column.dataType())) { for (int i = 0; i < num; i++) { if (defColumn.readInteger() == maxDefLevel) { } else if (DecimalType.is64BitDecimalType(column.dataType())) { for (int i = 0; i < num; i++) { if (defColumn.readInteger() == maxDefLevel) { } else if (DecimalType.isByteArrayDecimalType(column.dataType())) { for (int i = 0; i < num; i++) { if (defColumn.readInteger() == maxDefLevel) { throw new UnsupportedOperationException("Unimplemented type: " + column.dataType());
switch (descriptor.getType()) { case INT32: if (column.dataType() == DataTypes.IntegerType || DecimalType.is32BitDecimalType(column.dataType())) { for (int i = rowId; i < rowId + num; ++i) { if (!column.isNullAt(i)) { } else if (column.dataType() == DataTypes.ByteType) { for (int i = rowId; i < rowId + num; ++i) { if (!column.isNullAt(i)) { } else if (column.dataType() == DataTypes.ShortType) { for (int i = rowId; i < rowId + num; ++i) { if (!column.isNullAt(i)) { throw new UnsupportedOperationException("Unimplemented type: " + column.dataType()); if (column.dataType() == DataTypes.LongType || DecimalType.is64BitDecimalType(column.dataType())) { for (int i = rowId; i < rowId + num; ++i) { if (!column.isNullAt(i)) { } else if (column.dataType() == DataTypes.TimestampType) { for (int i = rowId; i < rowId + num; ++i) { if (!column.isNullAt(i)) { throw new UnsupportedOperationException("Unimplemented type: " + column.dataType()); if (column.dataType() == DataTypes.TimestampType) {
@Override public void update(int ordinal, Object value) { if (value == null) { setNullAt(ordinal); } else { DataType dt = columns[ordinal].dataType(); if (dt instanceof BooleanType) { setBoolean(ordinal, (boolean) value); } else if (dt instanceof IntegerType) { setInt(ordinal, (int) value); } else if (dt instanceof ShortType) { setShort(ordinal, (short) value); } else if (dt instanceof LongType) { setLong(ordinal, (long) value); } else if (dt instanceof FloatType) { setFloat(ordinal, (float) value); } else if (dt instanceof DoubleType) { setDouble(ordinal, (double) value); } else if (dt instanceof DecimalType) { DecimalType t = (DecimalType) dt; setDecimal(ordinal, Decimal.apply((BigDecimal) value, t.precision(), t.scale()), t.precision()); } else { throw new UnsupportedOperationException("Datatype not supported " + dt); } } }
private void readDoubleBatch(int rowId, int num, ColumnVector column) throws IOException { // This is where we implement support for the valid type conversions. // TODO: implement remaining type conversions if (column.dataType() == DataTypes.DoubleType) { defColumn.readDoubles( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); } else { throw new UnsupportedOperationException("Unimplemented type: " + column.dataType()); } }
/** * For all the read*Batch functions, reads `num` values from this columnReader into column. It * is guaranteed that num is smaller than the number of values left in the current page. */ private void readBooleanBatch(int rowId, int num, ColumnVector column) throws IOException { assert(column.dataType() == DataTypes.BooleanType); defColumn.readBooleans( num, column, rowId, maxDefLevel, (VectorizedValuesReader) dataColumn); }
(descriptor.getType() == PrimitiveType.PrimitiveTypeName.INT32 || (descriptor.getType() == PrimitiveType.PrimitiveTypeName.INT64 && column.dataType() != DataTypes.TimestampType) || descriptor.getType() == PrimitiveType.PrimitiveTypeName.FLOAT || descriptor.getType() == PrimitiveType.PrimitiveTypeName.DOUBLE ||
public final int appendNotNulls(int count) { assert (!(dataType() instanceof StructType)); reserve(elementsAppended + count); int result = elementsAppended; putNotNulls(elementsAppended, count); elementsAppended += count; return result; }
@Override public Object[] array() { DataType dt = data.dataType(); Object[] list = new Object[length];
row.setNullAt(i); } else { DataType dt = columns[i].dataType(); if (dt instanceof BooleanType) { row.setBoolean(i, getBoolean(i));
DataType t = col.dataType();