private Timestamp convert(Binary binary) { ByteBuffer buf = binary.toByteBuffer(); buf.order(ByteOrder.LITTLE_ENDIAN); long timeOfDayNanos = buf.getLong(); int julianDay = buf.getInt(); NanoTime nt = new NanoTime(julianDay, timeOfDayNanos); return NanoTimeUtils.getTimestamp(nt, skipTimestampConversion); }
/** * convert parquet binary decimal to BigDecimal, lifted from * https://github.com/apache/parquet-mr/blob/master/parquet-pig/src/main/java/org/apache/parquet/pig/convert/DecimalUtils.java#L38 */ private static BigDecimal convertBinaryToDecimal(Binary value, int precision, int scale) { // based on parquet-mr pig conversion which is based on spark conversion... yo dawg? if (precision <= 18) { ByteBuffer buffer = value.toByteBuffer(); byte[] bytes = buffer.array(); int start = buffer.arrayOffset() + buffer.position(); int end = buffer.arrayOffset() + buffer.limit(); long unscaled = 0L; int i = start; while (i < end) { unscaled = (unscaled << 8 | bytes[i] & 0xff); i++; } int bits = 8 * (end - start); long unscaledNew = (unscaled << (64 - bits)) >> (64 - bits); if (unscaledNew <= -Math.pow(10, 18) || unscaledNew >= Math.pow(10, 18)) { return new BigDecimal(unscaledNew); } else { return BigDecimal.valueOf(unscaledNew / Math.pow(10, scale)); } } else { return new BigDecimal(new BigInteger(value.getBytes()), scale); } }
public static NanoTime fromBinary(Binary bytes) { Preconditions.checkArgument(bytes.length() == 12, "Must be 12 bytes"); ByteBuffer buf = bytes.toByteBuffer(); buf.order(ByteOrder.LITTLE_ENDIAN); long timeOfDayNanos = buf.getLong(); int julianDay = buf.getInt(); return new NanoTime(julianDay, timeOfDayNanos); }
IntBuffer intBuf = intervalVal.toByteBuffer().order(ByteOrder.LITTLE_ENDIAN).asIntBuffer(); int months = intBuf.get(0); int days = intBuf.get(1);
@Override public ByteBuffer readBinary() throws TException { return value.toByteBuffer(); } });
@Override public UUID read(UUID ignored) { ByteBuffer buffer = column.nextBinary().toByteBuffer(); buffer.order(ByteOrder.BIG_ENDIAN); long mostSigBits = buffer.getLong(); long leastSigBits = buffer.getLong(); return new UUID(mostSigBits, leastSigBits); } }
public static NanoTime fromInt96(Int96Value int96) { ByteBuffer buf = int96.getInt96().toByteBuffer(); return new NanoTime(buf.getInt(), buf.getLong()); }
public static NanoTime fromBinary(Binary bytes) { Preconditions.checkArgument(bytes.length() == 12, "Must be 12 bytes"); ByteBuffer buf = bytes.toByteBuffer(); buf.order(ByteOrder.LITTLE_ENDIAN); long timeOfDayNanos = buf.getLong(); int julianDay = buf.getInt(); return new NanoTime(julianDay, timeOfDayNanos); }
public static NanoTime fromBinary(Binary bytes) { Preconditions.checkArgument(bytes.length() == 12, "Must be 12 bytes"); ByteBuffer buf = bytes.toByteBuffer(); buf.order(ByteOrder.LITTLE_ENDIAN); long timeOfDayNanos = buf.getLong(); int julianDay = buf.getInt(); return new NanoTime(julianDay, timeOfDayNanos); }
@Override public UTF8String read(UTF8String ignored) { Binary binary = column.nextBinary(); ByteBuffer buffer = binary.toByteBuffer(); if (buffer.hasArray()) { return UTF8String.fromBytes( buffer.array(), buffer.arrayOffset() + buffer.position(), buffer.remaining()); } else { return UTF8String.fromBytes(binary.getBytes()); } } }
@Override public void addBinary(Binary value) { holder.buffer.setBytes(0, value.toByteBuffer()); writer.writeVarBinary(holder.start, holder.end, holder.buffer); } }
@Override public byte[] read(byte[] reuse) { if (reuse != null) { column.nextBinary().toByteBuffer().duplicate().get(reuse); return reuse; } else { return column.nextBinary().getBytes(); } } }
@Override public void addBinary(Binary value) { holder.buffer.setBytes(0, value.toByteBuffer()); writer.write(holder); } }
@Override public Fixed read(Fixed reuse) { Fixed fixed; if (reuse != null) { fixed = reuse; } else { fixed = new Fixed(schema); } column.nextBinary().toByteBuffer().get(fixed.bytes()); return fixed; } }
@Override public boolean setSafe(int index, ArrowBuf value, int start, int length) { if (index >= vector.getValueCapacity()) { return false; } if (usingDictionary) { ByteBuffer buf = currDictValToWrite.toByteBuffer(); vector.setSafe(index, buf, buf.position(), currDictValToWrite.length()); } else { vector.setSafe(index, 1, start, start + length, value); } return true; }
@Override public boolean setSafe(int index, ArrowBuf value, int start, int length) { if (index >= nullableVarBinaryVector.getValueCapacity()) { return false; } if (usingDictionary) { ByteBuffer buf = currDictValToWrite.toByteBuffer(); nullableVarBinaryVector.setSafe(index, buf, buf.position(), currDictValToWrite.length()); } else { nullableVarBinaryVector.setSafe(index, 1, start, start + length, value); } return true; }
@Override public void addBinary(Binary value) { holder.buffer = buf = buf.reallocIfNeeded(value.length()); buf.setBytes(0, value.toByteBuffer()); holder.start = 0; holder.end = value.length(); writer.writeVarChar(holder.start, holder.end, holder.buffer); } }
@Override public void addBinary(Binary value) { holder.buffer = buf = buf.reallocIfNeeded(value.length()); buf.setBytes(0, value.toByteBuffer()); holder.start = 0; holder.end = value.length(); writer.writeVarBinary(holder.start, holder.end, holder.buffer); } }
@Override public void addBinary(Binary value) { holder.buffer = buf.reallocIfNeeded(value.length()); holder.buffer.setBytes(0, value.toByteBuffer()); holder.start = 0; holder.end = value.length(); writer.write(holder); }
/** * Returns the UTF8String for rowId. */ public final UTF8String getUTF8String(int rowId) { if (dictionary == null) { ColumnVector.Array a = getByteArray(rowId); return UTF8String.fromBytes(a.byteArray, a.byteArrayOffset, a.length); } else { Binary v = dictionary.decodeToBinary(dictionaryIds.getDictId(rowId)); return org.apache.spark.util.Utils.stringFromBuffer(v.toByteBuffer()); } }