@Override public void write(Object value) { byte[] vBinary = inspector.getPrimitiveJavaObject(value); recordConsumer.addBinary(Binary.fromByteArray(vBinary)); } }
private Binary decimalToBinary(final HiveDecimal hiveDecimal, final DecimalTypeInfo decimalTypeInfo) { int prec = decimalTypeInfo.precision(); int scale = decimalTypeInfo.scale(); byte[] decimalBytes = hiveDecimal.bigIntegerBytesScaled(scale); // Estimated number of bytes needed. int precToBytes = ParquetHiveSerDe.PRECISION_TO_BYTE_COUNT[prec - 1]; if (precToBytes == decimalBytes.length) { // No padding needed. return Binary.fromByteArray(decimalBytes); } byte[] tgt = new byte[precToBytes]; if (hiveDecimal.signum() == -1) { // For negative number, initializing bits to 1 for (int i = 0; i < precToBytes; i++) { tgt[i] |= 0xFF; } } System.arraycopy(decimalBytes, 0, tgt, precToBytes - decimalBytes.length, decimalBytes.length); // Padding leading zeroes/ones. return Binary.fromByteArray(tgt); } }
@Override public void writeValue() throws IOException { reader.read(holder); IntervalUtility.intToLEByteArray(holder.value, output, 0); Arrays.fill(output, 4, 8, (byte) 0); Arrays.fill(output, 8, 12, (byte) 0); consumer.addBinary(Binary.fromByteArray(output)); }
@Override public void writeValue() throws IOException { reader.read(holder); Arrays.fill(output, 0, 4, (byte) 0); IntervalUtility.intToLEByteArray(holder.days, output, 4); IntervalUtility.intToLEByteArray(holder.milliseconds, output, 8); consumer.addBinary(Binary.fromByteArray(output)); }
@Override public void writeField() throws IOException { consumer.startField(fieldName, fieldId); reader.read(holder); IntervalUtility.intToLEByteArray(holder.value, output, 0); Arrays.fill(output, 4, 8, (byte) 0); Arrays.fill(output, 8, 12, (byte) 0); consumer.addBinary(Binary.fromByteArray(output)); consumer.endField(fieldName, fieldId); } }
private Date getDateFromInt96(Object val) { if (val instanceof byte[]) { byte[] bytes = (byte[]) val; ByteBuffer buf = Binary.fromByteArray(bytes).toByteBuffer(); buf.order(ByteOrder.LITTLE_ENDIAN); long timeOfDayNanos = buf.getLong(); int julianDay = buf.getInt(); JDateTime date = new JDateTime(new JulianDateStamp(julianDay, (double) timeOfDayNanos / NANOS_PER_DAY)); return date.convertToDate(); } return null; } }
@Override public void writeField() throws IOException { consumer.startField(fieldName, fieldId); reader.read(holder); byte[] bytes = DecimalUtility.getBigDecimalFromSparse( holder.buffer, holder.start, Decimal38SparseHolder.nDecimalDigits, holder.scale).unscaledValue().toByteArray(); byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.DECIMAL38SPARSE)]; if (holder.getSign(holder.start, holder.buffer)) { Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF); } else { Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0); } System.arraycopy(bytes, 0, output, output.length - bytes.length, bytes.length); consumer.addBinary(Binary.fromByteArray(output)); consumer.endField(fieldName, fieldId); } }
@Override public void writeField() throws IOException { consumer.startField(fieldName, fieldId); reader.read(holder); Arrays.fill(output, 0, 4, (byte) 0); IntervalUtility.intToLEByteArray(holder.days, output, 4); IntervalUtility.intToLEByteArray(holder.milliseconds, output, 8); consumer.addBinary(Binary.fromByteArray(output)); consumer.endField(fieldName, fieldId); } }
@Override public void writeField() throws IOException { consumer.startField(fieldName, fieldId); reader.read(holder); byte[] bytes = DecimalUtility.getBigDecimalFromSparse( holder.buffer, holder.start, Decimal28SparseHolder.nDecimalDigits, holder.scale).unscaledValue().toByteArray(); byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.DECIMAL28SPARSE)]; if (holder.getSign(holder.start, holder.buffer)) { Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF); } else { Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0); } System.arraycopy(bytes, 0, output, output.length - bytes.length, bytes.length); consumer.addBinary(Binary.fromByteArray(output)); consumer.endField(fieldName, fieldId); } }
@Override public void writeField() throws IOException { if (!reader.isSet()) { return; } consumer.startField(fieldName, fieldId); reader.read(holder); IntervalUtility.intToLEByteArray(holder.value, output, 0); Arrays.fill(output, 4, 8, (byte) 0); Arrays.fill(output, 8, 12, (byte) 0); consumer.addBinary(Binary.fromByteArray(output)); consumer.endField(fieldName, fieldId); } }
@Override public void writeField() throws IOException { if (!reader.isSet()) { return; } consumer.startField(fieldName, fieldId); reader.read(holder); byte[] bytes = DecimalUtility.getBigDecimalFromSparse( holder.buffer, holder.start, Decimal38SparseHolder.nDecimalDigits, holder.scale).unscaledValue().toByteArray(); byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.DECIMAL38SPARSE)]; if (holder.getSign(holder.start, holder.buffer)) { Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF); } else { Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0); } System.arraycopy(bytes, 0, output, output.length - bytes.length, bytes.length); consumer.addBinary(Binary.fromByteArray(output)); consumer.endField(fieldName, fieldId); } }
@Override public void writeField() throws IOException { consumer.startField(fieldName, fieldId); reader.read(holder); IntervalUtility.intToLEByteArray(holder.months, output, 0); IntervalUtility.intToLEByteArray(holder.days, output, 4); IntervalUtility.intToLEByteArray(holder.milliseconds, output, 8); consumer.addBinary(Binary.fromByteArray(output)); consumer.endField(fieldName, fieldId); } }
@Override public void writeValue() throws IOException { /* Decimals are now Little Endian. So after reading the vector contents into holder, * we need to swap the bytes to get BE byte order. Copy the bytes from holder's * buffer and swap them before writing the binary. */ reader.read(holder); holder.buffer.getBytes(holder.start, bytes, 0, 16); DecimalHelper.swapBytes(bytes); consumer.addBinary(Binary.fromByteArray(bytes)); }
@Override public void writeField() throws IOException { if (!reader.isSet()) { return; } consumer.startField(fieldName, fieldId); reader.read(holder); byte[] bytes = DecimalUtility.getBigDecimalFromSparse( holder.buffer, holder.start, Decimal28SparseHolder.nDecimalDigits, holder.scale).unscaledValue().toByteArray(); byte[] output = new byte[ParquetTypeHelper.getLengthForMinorType(MinorType.DECIMAL28SPARSE)]; if (holder.getSign(holder.start, holder.buffer)) { Arrays.fill(output, 0, output.length - bytes.length, (byte)0xFF); } else { Arrays.fill(output, 0, output.length - bytes.length, (byte)0x0); } System.arraycopy(bytes, 0, output, output.length - bytes.length, bytes.length); consumer.addBinary(Binary.fromByteArray(output)); consumer.endField(fieldName, fieldId); } }
@Override public void writeField() throws IOException { if (!reader.isSet()) { return; } consumer.startField(fieldName, fieldId); reader.read(holder); Arrays.fill(output, 0, 4, (byte) 0); IntervalUtility.intToLEByteArray(holder.days, output, 4); IntervalUtility.intToLEByteArray(holder.milliseconds, output, 8); consumer.addBinary(Binary.fromByteArray(output)); consumer.endField(fieldName, fieldId); } }
@Test public void testInvalidBinaryLength() { try { byte[] invalidLengthBinaryTimestamp = new byte[8]; getTimestampMillis(Binary.fromByteArray(invalidLengthBinaryTimestamp)); } catch (PrestoException e) { assertEquals(e.getErrorCode(), NOT_SUPPORTED.toErrorCode()); assertEquals(e.getMessage(), "Parquet timestamp must be 12 bytes, actual 8"); } }
@Override public void writeField() throws IOException { if (!reader.isSet()) { return; } consumer.startField(fieldName, fieldId); reader.read(holder); IntervalUtility.intToLEByteArray(holder.months, output, 0); IntervalUtility.intToLEByteArray(holder.days, output, 4); IntervalUtility.intToLEByteArray(holder.milliseconds, output, 8); consumer.addBinary(Binary.fromByteArray(output)); consumer.endField(fieldName, fieldId); } }
@Override public void writeField() throws IOException { // empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements if (reader.size() == 0) { return; } consumer.startField(fieldName, fieldId); for (int i = 0; i < reader.size(); i++) { consumer.startField(fieldName, fieldId); reader.read(holder); IntervalUtility.intToLEByteArray(holder.value, output, 0); Arrays.fill(output, 4, 8, (byte) 0); Arrays.fill(output, 8, 12, (byte) 0); consumer.addBinary(Binary.fromByteArray(output)); consumer.endField(fieldName, fieldId); } consumer.endField(fieldName, fieldId); } }
@Override public void writeField() throws IOException { // empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements if (reader.size() == 0) { return; } consumer.startField(fieldName, fieldId); for (int i = 0; i < reader.size(); i++) { consumer.startField(fieldName, fieldId); reader.read(holder); Arrays.fill(output, 0, 4, (byte) 0); IntervalUtility.intToLEByteArray(holder.days, output, 4); IntervalUtility.intToLEByteArray(holder.milliseconds, output, 8); consumer.addBinary(Binary.fromByteArray(output)); consumer.endField(fieldName, fieldId); } consumer.endField(fieldName, fieldId); } }
@Override public void writeField() throws IOException { // empty lists are represented by simply not starting a field, rather than starting one and putting in 0 elements if (reader.size() == 0) { return; } consumer.startField(fieldName, fieldId); for (int i = 0; i < reader.size(); i++) { consumer.startField(fieldName, fieldId); reader.read(holder); IntervalUtility.intToLEByteArray(holder.months, output, 0); IntervalUtility.intToLEByteArray(holder.days, output, 4); IntervalUtility.intToLEByteArray(holder.milliseconds, output, 8); consumer.addBinary(Binary.fromByteArray(output)); consumer.endField(fieldName, fieldId); } consumer.endField(fieldName, fieldId); } }