/** * Inserts a key-value record into the sorter. If the sorter no longer has enough memory to hold * the record, the sorter sorts the existing records in-memory, writes them out as partially * sorted runs, and then reallocates memory to hold the new record. */ public void insertKV(UnsafeRow key, UnsafeRow value) throws IOException { final UnsafeExternalRowSorter.PrefixComputer.Prefix prefix = prefixComputer.computePrefix(key); sorter.insertKVRecord( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes(), prefix.value, prefix.isNull); }
/** * Inserts a key-value record into the sorter. If the sorter no longer has enough memory to hold * the record, the sorter sorts the existing records in-memory, writes them out as partially * sorted runs, and then reallocates memory to hold the new record. */ public void insertKV(UnsafeRow key, UnsafeRow value) throws IOException { final UnsafeExternalRowSorter.PrefixComputer.Prefix prefix = prefixComputer.computePrefix(key); sorter.insertKVRecord( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes(), prefix.value, prefix.isNull); }
/** * Inserts a key-value record into the sorter. If the sorter no longer has enough memory to hold * the record, the sorter sorts the existing records in-memory, writes them out as partially * sorted runs, and then reallocates memory to hold the new record. */ public void insertKV(UnsafeRow key, UnsafeRow value) throws IOException { final UnsafeExternalRowSorter.PrefixComputer.Prefix prefix = prefixComputer.computePrefix(key); sorter.insertKVRecord( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes(), prefix.value, prefix.isNull); }
public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key, int hash) { // Probe our map using the serialized key final BytesToBytesMap.Location loc = map.lookup( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), hash); if (!loc.isDefined()) { // This is the first time that we've seen this grouping key, so we'll insert a copy of the // empty aggregation buffer into the map: boolean putSucceeded = loc.append( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), emptyAggregationBuffer, Platform.BYTE_ARRAY_OFFSET, emptyAggregationBuffer.length ); if (!putSucceeded) { return null; } } // Reset the pointer to point to the value that we just stored or looked up: currentAggregationBuffer.pointTo( loc.getValueBase(), loc.getValueOffset(), loc.getValueLength() ); return currentAggregationBuffer; }
public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key, int hash) { // Probe our map using the serialized key final BytesToBytesMap.Location loc = map.lookup( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), hash); if (!loc.isDefined()) { // This is the first time that we've seen this grouping key, so we'll insert a copy of the // empty aggregation buffer into the map: boolean putSucceeded = loc.append( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), emptyAggregationBuffer, Platform.BYTE_ARRAY_OFFSET, emptyAggregationBuffer.length ); if (!putSucceeded) { return null; } } // Reset the pointer to point to the value that we just stored or looked up: currentAggregationBuffer.pointTo( loc.getValueBase(), loc.getValueOffset(), loc.getValueLength() ); return currentAggregationBuffer; }
public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key, int hash) { // Probe our map using the serialized key final BytesToBytesMap.Location loc = map.lookup( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), hash); if (!loc.isDefined()) { // This is the first time that we've seen this grouping key, so we'll insert a copy of the // empty aggregation buffer into the map: boolean putSucceeded = loc.append( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), emptyAggregationBuffer, Platform.BYTE_ARRAY_OFFSET, emptyAggregationBuffer.length ); if (!putSucceeded) { return null; } } // Reset the pointer to point to the value that we just stored or looked up: currentAggregationBuffer.pointTo( loc.getValueBase(), loc.getValueOffset(), loc.getValueLength() ); return currentAggregationBuffer; }
private void insertRow(UnsafeRow row) { Object recordBase = row.getBaseObject(); long recordOffset = row.getBaseOffset(); int recordLength = row.getSizeInBytes(); Object baseObject = dataPage.getBaseObject(); assert(pageCursor + recordLength <= dataPage.getBaseOffset() + dataPage.size()); long recordAddress = memoryManager.encodePageNumberAndOffset(dataPage, pageCursor); UnsafeAlignedOffset.putSize(baseObject, pageCursor, recordLength); pageCursor += uaoSize; Platform.copyMemory(recordBase, recordOffset, baseObject, pageCursor, recordLength); pageCursor += recordLength; assert(pos < 2); array.set(pos, recordAddress); pos++; }
private void insertRow(UnsafeRow row) { Object recordBase = row.getBaseObject(); long recordOffset = row.getBaseOffset(); int recordLength = row.getSizeInBytes(); Object baseObject = dataPage.getBaseObject(); assert(pageCursor + recordLength <= dataPage.getBaseOffset() + dataPage.size()); long recordAddress = memoryManager.encodePageNumberAndOffset(dataPage, pageCursor); UnsafeAlignedOffset.putSize(baseObject, pageCursor, recordLength); pageCursor += uaoSize; Platform.copyMemory(recordBase, recordOffset, baseObject, pageCursor, recordLength); pageCursor += recordLength; assert(pos < 2); array.set(pos, recordAddress); pos++; }
private void insertRow(UnsafeRow row) { Object recordBase = row.getBaseObject(); long recordOffset = row.getBaseOffset(); int recordLength = row.getSizeInBytes(); Object baseObject = dataPage.getBaseObject(); assert(pageCursor + recordLength <= dataPage.getBaseOffset() + dataPage.size()); long recordAddress = memoryManager.encodePageNumberAndOffset(dataPage, pageCursor); UnsafeAlignedOffset.putSize(baseObject, pageCursor, recordLength); pageCursor += uaoSize; Platform.copyMemory(recordBase, recordOffset, baseObject, pageCursor, recordLength); pageCursor += recordLength; assert(pos < 2); array.set(pos, recordAddress); pos++; }
@Test public void testBinaryComparatorForArrayColumn() throws Exception { int numFields = 1; UnsafeRow row1 = new UnsafeRow(numFields); byte[] data1 = new byte[100]; UnsafeArrayData arrayData1 = UnsafeArrayData.fromPrimitiveArray(new int[]{11, 42, -1}); row1.pointTo(data1, computeSizeInBytes(numFields * 8 + arrayData1.getSizeInBytes())); row1.setLong(0, (relativeOffset(numFields) << 32) | (long) arrayData1.getSizeInBytes()); Platform.copyMemory(arrayData1.getBaseObject(), arrayData1.getBaseOffset(), data1, row1.getBaseOffset() + relativeOffset(numFields), arrayData1.getSizeInBytes()); UnsafeRow row2 = new UnsafeRow(numFields); byte[] data2 = new byte[100]; UnsafeArrayData arrayData2 = UnsafeArrayData.fromPrimitiveArray(new int[]{22}); row2.pointTo(data2, computeSizeInBytes(numFields * 8 + arrayData2.getSizeInBytes())); row2.setLong(0, (relativeOffset(numFields) << 32) | (long) arrayData2.getSizeInBytes()); Platform.copyMemory(arrayData2.getBaseObject(), arrayData2.getBaseOffset(), data2, row2.getBaseOffset() + relativeOffset(numFields), arrayData2.getSizeInBytes()); insertRow(row1); insertRow(row2); assert(compare(0, 0) == 0); assert(compare(0, 1) > 0); }
@Test public void testBinaryComparatorForArrayColumn() throws Exception { int numFields = 1; UnsafeRow row1 = new UnsafeRow(numFields); byte[] data1 = new byte[100]; UnsafeArrayData arrayData1 = UnsafeArrayData.fromPrimitiveArray(new int[]{11, 42, -1}); row1.pointTo(data1, computeSizeInBytes(numFields * 8 + arrayData1.getSizeInBytes())); row1.setLong(0, (relativeOffset(numFields) << 32) | (long) arrayData1.getSizeInBytes()); Platform.copyMemory(arrayData1.getBaseObject(), arrayData1.getBaseOffset(), data1, row1.getBaseOffset() + relativeOffset(numFields), arrayData1.getSizeInBytes()); UnsafeRow row2 = new UnsafeRow(numFields); byte[] data2 = new byte[100]; UnsafeArrayData arrayData2 = UnsafeArrayData.fromPrimitiveArray(new int[]{22}); row2.pointTo(data2, computeSizeInBytes(numFields * 8 + arrayData2.getSizeInBytes())); row2.setLong(0, (relativeOffset(numFields) << 32) | (long) arrayData2.getSizeInBytes()); Platform.copyMemory(arrayData2.getBaseObject(), arrayData2.getBaseOffset(), data2, row2.getBaseOffset() + relativeOffset(numFields), arrayData2.getSizeInBytes()); insertRow(row1); insertRow(row2); assert(compare(0, 0) == 0); assert(compare(0, 1) > 0); }
@Test public void testBinaryComparatorForArrayColumn() throws Exception { int numFields = 1; UnsafeRow row1 = new UnsafeRow(numFields); byte[] data1 = new byte[100]; UnsafeArrayData arrayData1 = UnsafeArrayData.fromPrimitiveArray(new int[]{11, 42, -1}); row1.pointTo(data1, computeSizeInBytes(numFields * 8 + arrayData1.getSizeInBytes())); row1.setLong(0, (relativeOffset(numFields) << 32) | (long) arrayData1.getSizeInBytes()); Platform.copyMemory(arrayData1.getBaseObject(), arrayData1.getBaseOffset(), data1, row1.getBaseOffset() + relativeOffset(numFields), arrayData1.getSizeInBytes()); UnsafeRow row2 = new UnsafeRow(numFields); byte[] data2 = new byte[100]; UnsafeArrayData arrayData2 = UnsafeArrayData.fromPrimitiveArray(new int[]{22}); row2.pointTo(data2, computeSizeInBytes(numFields * 8 + arrayData2.getSizeInBytes())); row2.setLong(0, (relativeOffset(numFields) << 32) | (long) arrayData2.getSizeInBytes()); Platform.copyMemory(arrayData2.getBaseObject(), arrayData2.getBaseOffset(), data2, row2.getBaseOffset() + relativeOffset(numFields), arrayData2.getSizeInBytes()); insertRow(row1); insertRow(row2); assert(compare(0, 0) == 0); assert(compare(0, 1) > 0); }
/** * Inserts a key-value record into the sorter. If the sorter no longer has enough memory to hold * the record, the sorter sorts the existing records in-memory, writes them out as partially * sorted runs, and then reallocates memory to hold the new record. */ public void insertKV(UnsafeRow key, UnsafeRow value) throws IOException { final UnsafeExternalRowSorter.PrefixComputer.Prefix prefix = prefixComputer.computePrefix(key); sorter.insertKVRecord( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes(), prefix.value, prefix.isNull); }
@Test public void testBinaryComparatorForMixedColumns() throws Exception { int numFields = 4; UnsafeRow row1 = new UnsafeRow(numFields); byte[] data1 = new byte[100]; UTF8String str1 = UTF8String.fromString("Milk tea"); row1.pointTo(data1, computeSizeInBytes(numFields * 8 + str1.numBytes())); row1.setInt(0, 11); row1.setDouble(1, 3.14); row1.setInt(2, -1); row1.setLong(3, (relativeOffset(numFields) << 32) | (long) str1.numBytes()); Platform.copyMemory(str1.getBaseObject(), str1.getBaseOffset(), data1, row1.getBaseOffset() + relativeOffset(numFields), str1.numBytes()); UnsafeRow row2 = new UnsafeRow(numFields); byte[] data2 = new byte[100]; UTF8String str2 = UTF8String.fromString("Java"); row2.pointTo(data2, computeSizeInBytes(numFields * 8 + str2.numBytes())); row2.setInt(0, 11); row2.setDouble(1, 3.14); row2.setInt(2, -1); row2.setLong(3, (relativeOffset(numFields) << 32) | (long) str2.numBytes()); Platform.copyMemory(str2.getBaseObject(), str2.getBaseOffset(), data2, row2.getBaseOffset() + relativeOffset(numFields), str2.numBytes()); insertRow(row1); insertRow(row2); assert(compare(0, 0) == 0); assert(compare(0, 1) > 0); }
@Test public void testBinaryComparatorForMixedColumns() throws Exception { int numFields = 4; UnsafeRow row1 = new UnsafeRow(numFields); byte[] data1 = new byte[100]; UTF8String str1 = UTF8String.fromString("Milk tea"); row1.pointTo(data1, computeSizeInBytes(numFields * 8 + str1.numBytes())); row1.setInt(0, 11); row1.setDouble(1, 3.14); row1.setInt(2, -1); row1.setLong(3, (relativeOffset(numFields) << 32) | (long) str1.numBytes()); Platform.copyMemory(str1.getBaseObject(), str1.getBaseOffset(), data1, row1.getBaseOffset() + relativeOffset(numFields), str1.numBytes()); UnsafeRow row2 = new UnsafeRow(numFields); byte[] data2 = new byte[100]; UTF8String str2 = UTF8String.fromString("Java"); row2.pointTo(data2, computeSizeInBytes(numFields * 8 + str2.numBytes())); row2.setInt(0, 11); row2.setDouble(1, 3.14); row2.setInt(2, -1); row2.setLong(3, (relativeOffset(numFields) << 32) | (long) str2.numBytes()); Platform.copyMemory(str2.getBaseObject(), str2.getBaseOffset(), data2, row2.getBaseOffset() + relativeOffset(numFields), str2.numBytes()); insertRow(row1); insertRow(row2); assert(compare(0, 0) == 0); assert(compare(0, 1) > 0); }
@Test public void testBinaryComparatorForMixedColumns() throws Exception { int numFields = 4; UnsafeRow row1 = new UnsafeRow(numFields); byte[] data1 = new byte[100]; UTF8String str1 = UTF8String.fromString("Milk tea"); row1.pointTo(data1, computeSizeInBytes(numFields * 8 + str1.numBytes())); row1.setInt(0, 11); row1.setDouble(1, 3.14); row1.setInt(2, -1); row1.setLong(3, (relativeOffset(numFields) << 32) | (long) str1.numBytes()); Platform.copyMemory(str1.getBaseObject(), str1.getBaseOffset(), data1, row1.getBaseOffset() + relativeOffset(numFields), str1.numBytes()); UnsafeRow row2 = new UnsafeRow(numFields); byte[] data2 = new byte[100]; UTF8String str2 = UTF8String.fromString("Java"); row2.pointTo(data2, computeSizeInBytes(numFields * 8 + str2.numBytes())); row2.setInt(0, 11); row2.setDouble(1, 3.14); row2.setInt(2, -1); row2.setLong(3, (relativeOffset(numFields) << 32) | (long) str2.numBytes()); Platform.copyMemory(str2.getBaseObject(), str2.getBaseOffset(), data2, row2.getBaseOffset() + relativeOffset(numFields), str2.numBytes()); insertRow(row1); insertRow(row2); assert(compare(0, 0) == 0); assert(compare(0, 1) > 0); }
public void insertRow(UnsafeRow row) throws IOException { final PrefixComputer.Prefix prefix = prefixComputer.computePrefix(row); sorter.insertRecord( row.getBaseObject(), row.getBaseOffset(), row.getSizeInBytes(), prefix.value, prefix.isNull ); numRowsInserted++; if (testSpillFrequency > 0 && (numRowsInserted % testSpillFrequency) == 0) { sorter.spill(); } }
private UnsafeRow appendRow(RowBasedKeyValueBatch batch, UnsafeRow key, UnsafeRow value) { return batch.appendRow(key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes()); }
private UnsafeRow appendRow(RowBasedKeyValueBatch batch, UnsafeRow key, UnsafeRow value) { return batch.appendRow(key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes()); }
private UnsafeRow appendRow(RowBasedKeyValueBatch batch, UnsafeRow key, UnsafeRow value) { return batch.appendRow(key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes()); }