/** * Inserts a key-value record into the sorter. If the sorter no longer has enough memory to hold * the record, the sorter sorts the existing records in-memory, writes them out as partially * sorted runs, and then reallocates memory to hold the new record. */ public void insertKV(UnsafeRow key, UnsafeRow value) throws IOException { final UnsafeExternalRowSorter.PrefixComputer.Prefix prefix = prefixComputer.computePrefix(key); sorter.insertKVRecord( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes(), prefix.value, prefix.isNull); }
/** * Inserts a key-value record into the sorter. If the sorter no longer has enough memory to hold * the record, the sorter sorts the existing records in-memory, writes them out as partially * sorted runs, and then reallocates memory to hold the new record. */ public void insertKV(UnsafeRow key, UnsafeRow value) throws IOException { final UnsafeExternalRowSorter.PrefixComputer.Prefix prefix = prefixComputer.computePrefix(key); sorter.insertKVRecord( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes(), prefix.value, prefix.isNull); }
/** * Inserts a key-value record into the sorter. If the sorter no longer has enough memory to hold * the record, the sorter sorts the existing records in-memory, writes them out as partially * sorted runs, and then reallocates memory to hold the new record. */ public void insertKV(UnsafeRow key, UnsafeRow value) throws IOException { final UnsafeExternalRowSorter.PrefixComputer.Prefix prefix = prefixComputer.computePrefix(key); sorter.insertKVRecord( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes(), prefix.value, prefix.isNull); }
public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key, int hash) { // Probe our map using the serialized key final BytesToBytesMap.Location loc = map.lookup( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), hash); if (!loc.isDefined()) { // This is the first time that we've seen this grouping key, so we'll insert a copy of the // empty aggregation buffer into the map: boolean putSucceeded = loc.append( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), emptyAggregationBuffer, Platform.BYTE_ARRAY_OFFSET, emptyAggregationBuffer.length ); if (!putSucceeded) { return null; } } // Reset the pointer to point to the value that we just stored or looked up: currentAggregationBuffer.pointTo( loc.getValueBase(), loc.getValueOffset(), loc.getValueLength() ); return currentAggregationBuffer; }
public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key, int hash) { // Probe our map using the serialized key final BytesToBytesMap.Location loc = map.lookup( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), hash); if (!loc.isDefined()) { // This is the first time that we've seen this grouping key, so we'll insert a copy of the // empty aggregation buffer into the map: boolean putSucceeded = loc.append( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), emptyAggregationBuffer, Platform.BYTE_ARRAY_OFFSET, emptyAggregationBuffer.length ); if (!putSucceeded) { return null; } } // Reset the pointer to point to the value that we just stored or looked up: currentAggregationBuffer.pointTo( loc.getValueBase(), loc.getValueOffset(), loc.getValueLength() ); return currentAggregationBuffer; }
public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key, int hash) { // Probe our map using the serialized key final BytesToBytesMap.Location loc = map.lookup( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), hash); if (!loc.isDefined()) { // This is the first time that we've seen this grouping key, so we'll insert a copy of the // empty aggregation buffer into the map: boolean putSucceeded = loc.append( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), emptyAggregationBuffer, Platform.BYTE_ARRAY_OFFSET, emptyAggregationBuffer.length ); if (!putSucceeded) { return null; } } // Reset the pointer to point to the value that we just stored or looked up: currentAggregationBuffer.pointTo( loc.getValueBase(), loc.getValueOffset(), loc.getValueLength() ); return currentAggregationBuffer; }
private void insertRow(UnsafeRow row) { Object recordBase = row.getBaseObject(); long recordOffset = row.getBaseOffset(); int recordLength = row.getSizeInBytes(); Object baseObject = dataPage.getBaseObject(); assert(pageCursor + recordLength <= dataPage.getBaseOffset() + dataPage.size()); long recordAddress = memoryManager.encodePageNumberAndOffset(dataPage, pageCursor); UnsafeAlignedOffset.putSize(baseObject, pageCursor, recordLength); pageCursor += uaoSize; Platform.copyMemory(recordBase, recordOffset, baseObject, pageCursor, recordLength); pageCursor += recordLength; assert(pos < 2); array.set(pos, recordAddress); pos++; }
private void insertRow(UnsafeRow row) { Object recordBase = row.getBaseObject(); long recordOffset = row.getBaseOffset(); int recordLength = row.getSizeInBytes(); Object baseObject = dataPage.getBaseObject(); assert(pageCursor + recordLength <= dataPage.getBaseOffset() + dataPage.size()); long recordAddress = memoryManager.encodePageNumberAndOffset(dataPage, pageCursor); UnsafeAlignedOffset.putSize(baseObject, pageCursor, recordLength); pageCursor += uaoSize; Platform.copyMemory(recordBase, recordOffset, baseObject, pageCursor, recordLength); pageCursor += recordLength; assert(pos < 2); array.set(pos, recordAddress); pos++; }
private void insertRow(UnsafeRow row) { Object recordBase = row.getBaseObject(); long recordOffset = row.getBaseOffset(); int recordLength = row.getSizeInBytes(); Object baseObject = dataPage.getBaseObject(); assert(pageCursor + recordLength <= dataPage.getBaseOffset() + dataPage.size()); long recordAddress = memoryManager.encodePageNumberAndOffset(dataPage, pageCursor); UnsafeAlignedOffset.putSize(baseObject, pageCursor, recordLength); pageCursor += uaoSize; Platform.copyMemory(recordBase, recordOffset, baseObject, pageCursor, recordLength); pageCursor += recordLength; assert(pos < 2); array.set(pos, recordAddress); pos++; }
/** * Inserts a key-value record into the sorter. If the sorter no longer has enough memory to hold * the record, the sorter sorts the existing records in-memory, writes them out as partially * sorted runs, and then reallocates memory to hold the new record. */ public void insertKV(UnsafeRow key, UnsafeRow value) throws IOException { final UnsafeExternalRowSorter.PrefixComputer.Prefix prefix = prefixComputer.computePrefix(key); sorter.insertKVRecord( key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes(), prefix.value, prefix.isNull); }
public final void write(int ordinal, UnsafeRow row) { writeAlignedBytes(ordinal, row.getBaseObject(), row.getBaseOffset(), row.getSizeInBytes()); }
public final void write(int ordinal, UnsafeRow row) { writeAlignedBytes(ordinal, row.getBaseObject(), row.getBaseOffset(), row.getSizeInBytes()); }
/** * Returns the value row by two steps: * 1) looking up the key row with the same id (skipped if the key row is cached) * 2) retrieve the value row by reusing the metadata from step 1) * In most times, 1) is skipped because `getKeyRow(id)` is often called before `getValueRow(id)`. */ @Override public UnsafeRow getValueFromKey(int rowId) { if (keyRowId != rowId) { getKeyRow(rowId); } assert(rowId >= 0); long offset = keyRow.getBaseOffset(); int klen = keyRow.getSizeInBytes(); int vlen = Platform.getInt(base, offset - 8) - klen - 4; valueRow.pointTo(base, offset + klen, vlen); return valueRow; }
/** * Returns the value row by two steps: * 1) looking up the key row with the same id (skipped if the key row is cached) * 2) retrieve the value row by reusing the metadata from step 1) * In most times, 1) is skipped because `getKeyRow(id)` is often called before `getValueRow(id)`. */ @Override public UnsafeRow getValueFromKey(int rowId) { if (keyRowId != rowId) { getKeyRow(rowId); } assert(rowId >= 0); long offset = keyRow.getBaseOffset(); int klen = keyRow.getSizeInBytes(); int vlen = Platform.getInt(base, offset - 8) - klen - 4; valueRow.pointTo(base, offset + klen, vlen); return valueRow; }
public void insertRow(UnsafeRow row) throws IOException { final PrefixComputer.Prefix prefix = prefixComputer.computePrefix(row); sorter.insertRecord( row.getBaseObject(), row.getBaseOffset(), row.getSizeInBytes(), prefix.value, prefix.isNull ); numRowsInserted++; if (testSpillFrequency > 0 && (numRowsInserted % testSpillFrequency) == 0) { sorter.spill(); } }
public void insertRow(UnsafeRow row) throws IOException { final PrefixComputer.Prefix prefix = prefixComputer.computePrefix(row); sorter.insertRecord( row.getBaseObject(), row.getBaseOffset(), row.getSizeInBytes(), prefix.value, prefix.isNull ); numRowsInserted++; if (testSpillFrequency > 0 && (numRowsInserted % testSpillFrequency) == 0) { sorter.spill(); } }
public void insertRow(UnsafeRow row) throws IOException { final PrefixComputer.Prefix prefix = prefixComputer.computePrefix(row); sorter.insertRecord( row.getBaseObject(), row.getBaseOffset(), row.getSizeInBytes(), prefix.value, prefix.isNull ); numRowsInserted++; if (testSpillFrequency > 0 && (numRowsInserted % testSpillFrequency) == 0) { sorter.spill(); } }
private UnsafeRow appendRow(RowBasedKeyValueBatch batch, UnsafeRow key, UnsafeRow value) { return batch.appendRow(key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes()); }
private UnsafeRow appendRow(RowBasedKeyValueBatch batch, UnsafeRow key, UnsafeRow value) { return batch.appendRow(key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes()); }
private UnsafeRow appendRow(RowBasedKeyValueBatch batch, UnsafeRow key, UnsafeRow value) { return batch.appendRow(key.getBaseObject(), key.getBaseOffset(), key.getSizeInBytes(), value.getBaseObject(), value.getBaseOffset(), value.getSizeInBytes()); }