public final int appendArray(int length) { reserve(elementsAppended + 1); putArray(elementsAppended, arrayData().elementsAppended, length); return elementsAppended++; }
public final int appendArray(int length) { reserve(elementsAppended + 1); putArray(elementsAppended, arrayData().elementsAppended, length); return elementsAppended++; }
public final int appendByteArray(byte[] value, int offset, int length) { int copiedOffset = arrayData().appendBytes(length, value, offset); reserve(elementsAppended + 1); putArray(elementsAppended, copiedOffset, length); return elementsAppended++; }
public final int appendByteArray(byte[] value, int offset, int length) { int copiedOffset = arrayData().appendBytes(length, value, offset); reserve(elementsAppended + 1); putArray(elementsAppended, copiedOffset, length); return elementsAppended++; }
@Override public final ColumnarArray getArray(int rowId) { if (isNullAt(rowId)) return null; return new ColumnarArray(arrayData(), getArrayOffset(rowId), getArrayLength(rowId)); }
@Override public final ColumnarArray getArray(int rowId) { if (isNullAt(rowId)) return null; return new ColumnarArray(arrayData(), getArrayOffset(rowId), getArrayLength(rowId)); }
@Override public byte[] getBinary(int rowId) { if (isNullAt(rowId)) return null; if (dictionary == null) { return arrayData().getBytes(getArrayOffset(rowId), getArrayLength(rowId)); } else { return dictionary.decodeToBinary(dictionaryIds.getDictId(rowId)); } }
@Override public byte[] getBinary(int rowId) { if (isNullAt(rowId)) return null; if (dictionary == null) { return arrayData().getBytes(getArrayOffset(rowId), getArrayLength(rowId)); } else { return dictionary.decodeToBinary(dictionaryIds.getDictId(rowId)); } }
@Override public UTF8String getUTF8String(int rowId) { if (isNullAt(rowId)) return null; if (dictionary == null) { return arrayData().getBytesAsUTF8String(getArrayOffset(rowId), getArrayLength(rowId)); } else { byte[] bytes = dictionary.decodeToBinary(dictionaryIds.getDictId(rowId)); return UTF8String.fromBytes(bytes); } }
@Override public UTF8String getUTF8String(int rowId) { if (isNullAt(rowId)) return null; if (dictionary == null) { return arrayData().getBytesAsUTF8String(getArrayOffset(rowId), getArrayLength(rowId)); } else { byte[] bytes = dictionary.decodeToBinary(dictionaryIds.getDictId(rowId)); return UTF8String.fromBytes(bytes); } }
/** * Put `HiveDecimalWritable`s to a `WritableColumnVector`. */ private static void putDecimalWritables( WritableColumnVector toColumn, int size, int precision, int scale, HiveDecimalWritable decimalWritable) { HiveDecimal decimal = decimalWritable.getHiveDecimal(); Decimal value = Decimal.apply(decimal.bigDecimalValue(), decimal.precision(), decimal.scale()); value.changePrecision(precision, scale); if (precision <= Decimal.MAX_INT_DIGITS()) { toColumn.putInts(0, size, (int) value.toUnscaledLong()); } else if (precision <= Decimal.MAX_LONG_DIGITS()) { toColumn.putLongs(0, size, value.toUnscaledLong()); } else { byte[] bytes = value.toJavaBigDecimal().unscaledValue().toByteArray(); toColumn.arrayData().reserve(bytes.length); toColumn.arrayData().putBytes(0, bytes.length, bytes, 0); for (int index = 0; index < size; index++) { toColumn.putArray(index, 0, bytes.length); } } } }
/** * Put `HiveDecimalWritable`s to a `WritableColumnVector`. */ private static void putDecimalWritables( WritableColumnVector toColumn, int size, int precision, int scale, HiveDecimalWritable decimalWritable) { HiveDecimal decimal = decimalWritable.getHiveDecimal(); Decimal value = Decimal.apply(decimal.bigDecimalValue(), decimal.precision(), decimal.scale()); value.changePrecision(precision, scale); if (precision <= Decimal.MAX_INT_DIGITS()) { toColumn.putInts(0, size, (int) value.toUnscaledLong()); } else if (precision <= Decimal.MAX_LONG_DIGITS()) { toColumn.putLongs(0, size, value.toUnscaledLong()); } else { byte[] bytes = value.toJavaBigDecimal().unscaledValue().toByteArray(); toColumn.arrayData().reserve(bytes.length); toColumn.arrayData().putBytes(0, bytes.length, bytes, 0); for (int index = 0; index < size; index++) { toColumn.putArray(index, 0, bytes.length); } } } }
/** * Put a `HiveDecimalWritable` to a `WritableColumnVector`. */ private static void putDecimalWritable( WritableColumnVector toColumn, int index, int precision, int scale, HiveDecimalWritable decimalWritable) { HiveDecimal decimal = decimalWritable.getHiveDecimal(); Decimal value = Decimal.apply(decimal.bigDecimalValue(), decimal.precision(), decimal.scale()); value.changePrecision(precision, scale); if (precision <= Decimal.MAX_INT_DIGITS()) { toColumn.putInt(index, (int) value.toUnscaledLong()); } else if (precision <= Decimal.MAX_LONG_DIGITS()) { toColumn.putLong(index, value.toUnscaledLong()); } else { byte[] bytes = value.toJavaBigDecimal().unscaledValue().toByteArray(); toColumn.arrayData().putBytes(index * 16, bytes.length, bytes, 0); toColumn.putArray(index, index * 16, bytes.length); } }
/** * Put a `HiveDecimalWritable` to a `WritableColumnVector`. */ private static void putDecimalWritable( WritableColumnVector toColumn, int index, int precision, int scale, HiveDecimalWritable decimalWritable) { HiveDecimal decimal = decimalWritable.getHiveDecimal(); Decimal value = Decimal.apply(decimal.bigDecimalValue(), decimal.precision(), decimal.scale()); value.changePrecision(precision, scale); if (precision <= Decimal.MAX_INT_DIGITS()) { toColumn.putInt(index, (int) value.toUnscaledLong()); } else if (precision <= Decimal.MAX_LONG_DIGITS()) { toColumn.putLong(index, value.toUnscaledLong()); } else { byte[] bytes = value.toJavaBigDecimal().unscaledValue().toByteArray(); toColumn.arrayData().putBytes(index * 16, bytes.length, bytes, 0); toColumn.putArray(index, index * 16, bytes.length); } }
} else if (type instanceof StringType || type instanceof BinaryType) { BytesColumnVector data = ((BytesColumnVector)fromColumn); WritableColumnVector arrayData = toColumn.arrayData(); int totalNumBytes = IntStream.of(data.length).sum(); arrayData.reserve(totalNumBytes); DecimalColumnVector data = ((DecimalColumnVector)fromColumn); if (decimalType.precision() > Decimal.MAX_LONG_DIGITS()) { toColumn.arrayData().reserve(batchSize * 16);
} else if (type instanceof StringType || type instanceof BinaryType) { BytesColumnVector data = ((BytesColumnVector)fromColumn); WritableColumnVector arrayData = toColumn.arrayData(); int totalNumBytes = IntStream.of(data.length).sum(); arrayData.reserve(totalNumBytes); DecimalColumnVector data = ((DecimalColumnVector)fromColumn); if (decimalType.precision() > Decimal.MAX_LONG_DIGITS()) { toColumn.arrayData().reserve(batchSize * 16);
private static void appendValue(WritableColumnVector dst, DataType t, Row src, int fieldIdx) { if (t instanceof ArrayType) { ArrayType at = (ArrayType)t; if (src.isNullAt(fieldIdx)) { dst.appendNull(); } else { List<Object> values = src.getList(fieldIdx); dst.appendArray(values.size()); for (Object o : values) { appendValue(dst.arrayData(), at.elementType(), o); } } } else if (t instanceof StructType) { StructType st = (StructType)t; if (src.isNullAt(fieldIdx)) { dst.appendStruct(true); } else { dst.appendStruct(false); Row c = src.getStruct(fieldIdx); for (int i = 0; i < st.fields().length; i++) { appendValue(dst.getChild(i), st.fields()[i].dataType(), c, i); } } } else { appendValue(dst, t, src.get(fieldIdx)); } }
private static void appendValue(WritableColumnVector dst, DataType t, Row src, int fieldIdx) { if (t instanceof ArrayType) { ArrayType at = (ArrayType)t; if (src.isNullAt(fieldIdx)) { dst.appendNull(); } else { List<Object> values = src.getList(fieldIdx); dst.appendArray(values.size()); for (Object o : values) { appendValue(dst.arrayData(), at.elementType(), o); } } } else if (t instanceof StructType) { StructType st = (StructType)t; if (src.isNullAt(fieldIdx)) { dst.appendStruct(true); } else { dst.appendStruct(false); Row c = src.getStruct(fieldIdx); for (int i = 0; i < st.fields().length; i++) { appendValue(dst.getChild(i), st.fields()[i].dataType(), c, i); } } } else { appendValue(dst, t, src.get(fieldIdx)); } }