@Test public void testBinaryComparatorForArrayColumn() throws Exception { int numFields = 1; UnsafeRow row1 = new UnsafeRow(numFields); byte[] data1 = new byte[100]; UnsafeArrayData arrayData1 = UnsafeArrayData.fromPrimitiveArray(new int[]{11, 42, -1}); row1.pointTo(data1, computeSizeInBytes(numFields * 8 + arrayData1.getSizeInBytes())); row1.setLong(0, (relativeOffset(numFields) << 32) | (long) arrayData1.getSizeInBytes()); Platform.copyMemory(arrayData1.getBaseObject(), arrayData1.getBaseOffset(), data1, row1.getBaseOffset() + relativeOffset(numFields), arrayData1.getSizeInBytes()); UnsafeRow row2 = new UnsafeRow(numFields); byte[] data2 = new byte[100]; UnsafeArrayData arrayData2 = UnsafeArrayData.fromPrimitiveArray(new int[]{22}); row2.pointTo(data2, computeSizeInBytes(numFields * 8 + arrayData2.getSizeInBytes())); row2.setLong(0, (relativeOffset(numFields) << 32) | (long) arrayData2.getSizeInBytes()); Platform.copyMemory(arrayData2.getBaseObject(), arrayData2.getBaseOffset(), data2, row2.getBaseOffset() + relativeOffset(numFields), arrayData2.getSizeInBytes()); insertRow(row1); insertRow(row2); assert(compare(0, 0) == 0); assert(compare(0, 1) > 0); }
@Override public UnsafeArrayData getArray(int ordinal) { if (isNullAt(ordinal)) return null; final long offsetAndSize = getLong(ordinal); final int offset = (int) (offsetAndSize >> 32); final int size = (int) offsetAndSize; final UnsafeArrayData array = new UnsafeArrayData(); array.pointTo(baseObject, baseOffset + offset, size); return array; }
public static boolean shouldUseGenericArrayData(int elementSize, long length) { final long headerInBytes = calculateHeaderPortionInBytes(length); final long valueRegionInBytes = elementSize * length; final long totalSizeInLongs = (headerInBytes + valueRegionInBytes + 7) / 8; return totalSizeInLongs > Integer.MAX_VALUE / 8; }
@Override public Object get(int ordinal, DataType dataType) { if (isNullAt(ordinal) || dataType instanceof NullType) { return null; } else if (dataType instanceof BooleanType) { return getBoolean(ordinal); } else if (dataType instanceof ByteType) { return getByte(ordinal); } else if (dataType instanceof ShortType) { return getShort(ordinal); } else if (dataType instanceof IntegerType) { return getInt(ordinal); } else if (dataType instanceof LongType) { return getLong(ordinal); } else if (dataType instanceof FloatType) { return getFloat(ordinal); } else if (dataType instanceof DoubleType) { return getDouble(ordinal); } else if (dataType instanceof DecimalType) { DecimalType dt = (DecimalType) dataType; return getDecimal(ordinal, dt.precision(), dt.scale()); } else if (dataType instanceof DateType) { return getInt(ordinal); } else if (dataType instanceof TimestampType) { return getLong(ordinal); } else if (dataType instanceof BinaryType) { return getBinary(ordinal); } else if (dataType instanceof StringType) { return getUTF8String(ordinal); } else if (dataType instanceof CalendarIntervalType) {
@Override public short getShort(int ordinal) { assertIndexIsValid(ordinal); return Platform.getShort(baseObject, getElementOffset(ordinal, 2)); }
public static UnsafeArrayData fromPrimitiveArray(boolean[] arr) { return fromPrimitiveArray(arr, Platform.BOOLEAN_ARRAY_OFFSET, arr.length, 1); }
public final void write(UnsafeArrayData array) { // Unsafe arrays both can be written as a regular array field or as part of a map. This makes // updating the offset and size dependent on the code path, this is why we currently do not // provide an method for writing unsafe arrays that also updates the size and offset. int numBytes = array.getSizeInBytes(); grow(numBytes); Platform.copyMemory( array.getBaseObject(), array.getBaseOffset(), getBuffer(), cursor(), numBytes); increaseCursor(numBytes); }
public static UnsafeArrayData createFreshArray(int length, int elementSize) { final long headerInBytes = calculateHeaderPortionInBytes(length); final long valueRegionInBytes = (long)elementSize * length; final long totalSizeInLongs = (headerInBytes + valueRegionInBytes + 7) / 8; if (totalSizeInLongs > Integer.MAX_VALUE / 8) { throw new UnsupportedOperationException("Cannot convert this array to unsafe format as " + "it's too big."); } final long[] data = new long[(int)totalSizeInLongs]; Platform.putLong(data, Platform.LONG_ARRAY_OFFSET, length); UnsafeArrayData result = new UnsafeArrayData(); result.pointTo(data, Platform.LONG_ARRAY_OFFSET, (int)totalSizeInLongs * 8); return result; }
final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes(); ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor); holder.cursor += sizeInBytes1; final int sizeInBytes2 = ((UnsafeArrayData) values).getSizeInBytes(); ((UnsafeArrayData) values).writeToMemory(holder.buffer, holder.cursor); holder.cursor += sizeInBytes2;
@Override public UnsafeArrayData copy() { UnsafeArrayData arrayCopy = new UnsafeArrayData(); final byte[] arrayDataCopy = new byte[sizeInBytes]; Platform.copyMemory( baseObject, baseOffset, arrayDataCopy, Platform.BYTE_ARRAY_OFFSET, sizeInBytes); arrayCopy.pointTo(arrayDataCopy, Platform.BYTE_ARRAY_OFFSET, sizeInBytes); return arrayCopy; }
public void setNullAt(int ordinal) { assertIndexIsValid(ordinal); BitSetMethods.set(baseObject, baseOffset + 8, ordinal); /* we assume the corresponding column was already 0 or will be set to 0 later by the caller side */ }
public void writeTo(ByteBuffer buffer) { assert(buffer.hasArray()); byte[] target = buffer.array(); int offset = buffer.arrayOffset(); int pos = buffer.position(); writeToMemory(target, Platform.BYTE_ARRAY_OFFSET + offset + pos); buffer.position(pos + sizeInBytes); }
/** * Construct a new UnsafeMapData. The resulting UnsafeMapData won't be usable until * `pointTo()` has been called, since the value returned by this constructor is equivalent * to a null pointer. */ public UnsafeMapData() { keys = new UnsafeArrayData(); values = new UnsafeArrayData(); }
@Override public Object get(int ordinal, DataType dataType) { if (isNullAt(ordinal) || dataType instanceof NullType) { return null; } else if (dataType instanceof BooleanType) { return getBoolean(ordinal); } else if (dataType instanceof ByteType) { return getByte(ordinal); } else if (dataType instanceof ShortType) { return getShort(ordinal); } else if (dataType instanceof IntegerType) { return getInt(ordinal); } else if (dataType instanceof LongType) { return getLong(ordinal); } else if (dataType instanceof FloatType) { return getFloat(ordinal); } else if (dataType instanceof DoubleType) { return getDouble(ordinal); } else if (dataType instanceof DecimalType) { DecimalType dt = (DecimalType) dataType; return getDecimal(ordinal, dt.precision(), dt.scale()); } else if (dataType instanceof DateType) { return getInt(ordinal); } else if (dataType instanceof TimestampType) { return getLong(ordinal); } else if (dataType instanceof BinaryType) { return getBinary(ordinal); } else if (dataType instanceof StringType) { return getUTF8String(ordinal); } else if (dataType instanceof CalendarIntervalType) {
@Override public byte getByte(int ordinal) { assertIndexIsValid(ordinal); return Platform.getByte(baseObject, getElementOffset(ordinal, 1)); }
public static UnsafeArrayData fromPrimitiveArray(float[] arr) { return fromPrimitiveArray(arr, Platform.FLOAT_ARRAY_OFFSET, arr.length, 4); }
public final void write(UnsafeArrayData array) { // Unsafe arrays both can be written as a regular array field or as part of a map. This makes // updating the offset and size dependent on the code path, this is why we currently do not // provide an method for writing unsafe arrays that also updates the size and offset. int numBytes = array.getSizeInBytes(); grow(numBytes); Platform.copyMemory( array.getBaseObject(), array.getBaseOffset(), getBuffer(), cursor(), numBytes); increaseCursor(numBytes); }
public static UnsafeArrayData createFreshArray(int length, int elementSize) { final long headerInBytes = calculateHeaderPortionInBytes(length); final long valueRegionInBytes = (long)elementSize * length; final long totalSizeInLongs = (headerInBytes + valueRegionInBytes + 7) / 8; if (totalSizeInLongs > Integer.MAX_VALUE / 8) { throw new UnsupportedOperationException("Cannot convert this array to unsafe format as " + "it's too big."); } final long[] data = new long[(int)totalSizeInLongs]; Platform.putLong(data, Platform.LONG_ARRAY_OFFSET, length); UnsafeArrayData result = new UnsafeArrayData(); result.pointTo(data, Platform.LONG_ARRAY_OFFSET, (int)totalSizeInLongs * 8); return result; }
final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes(); ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor); holder.cursor += sizeInBytes1; final int sizeInBytes2 = ((UnsafeArrayData) values).getSizeInBytes(); ((UnsafeArrayData) values).writeToMemory(holder.buffer, holder.cursor); holder.cursor += sizeInBytes2;
@Override public UnsafeArrayData copy() { UnsafeArrayData arrayCopy = new UnsafeArrayData(); final byte[] arrayDataCopy = new byte[sizeInBytes]; Platform.copyMemory( baseObject, baseOffset, arrayDataCopy, Platform.BYTE_ARRAY_OFFSET, sizeInBytes); arrayCopy.pointTo(arrayDataCopy, Platform.BYTE_ARRAY_OFFSET, sizeInBytes); return arrayCopy; }