/** * Allocates columns to store elements of each field of the schema on heap. * Capacity is the initial capacity of the vector and it will grow as necessary. Capacity is * in number of elements, not number of bytes. */ public static OnHeapColumnVector[] allocateColumns(int capacity, StructType schema) { return allocateColumns(capacity, schema.fields()); }
/** * Allocates columns to store elements of each field of the schema on heap. * Capacity is the initial capacity of the vector and it will grow as necessary. Capacity is * in number of elements, not number of bytes. */ public static OnHeapColumnVector[] allocateColumns(int capacity, StructType schema) { return allocateColumns(capacity, schema.fields()); }
public AggregateHashMap(StructType schema, int capacity, double loadFactor, int maxSteps) { // We currently only support single key-value pair that are both longs assert (schema.size() == 2 && schema.fields()[0].dataType() == LongType && schema.fields()[1].dataType() == LongType); // capacity should be a power of 2 assert (capacity > 0 && ((capacity & (capacity - 1)) == 0)); this.maxSteps = maxSteps; numBuckets = (int) (capacity / loadFactor); columnVectors = OnHeapColumnVector.allocateColumns(capacity, schema); aggBufferRow = new MutableColumnarRow(columnVectors); buckets = new int[numBuckets]; Arrays.fill(buckets, -1); }
public AggregateHashMap(StructType schema, int capacity, double loadFactor, int maxSteps) { // We currently only support single key-value pair that are both longs assert (schema.size() == 2 && schema.fields()[0].dataType() == LongType && schema.fields()[1].dataType() == LongType); // capacity should be a power of 2 assert (capacity > 0 && ((capacity & (capacity - 1)) == 0)); this.maxSteps = maxSteps; numBuckets = (int) (capacity / loadFactor); columnVectors = OnHeapColumnVector.allocateColumns(capacity, schema); aggBufferRow = new MutableColumnarRow(columnVectors); buckets = new int[numBuckets]; Arrays.fill(buckets, -1); }
/** * Converts an iterator of rows into a single ColumnBatch. */ public static ColumnarBatch toBatch( StructType schema, MemoryMode memMode, Iterator<Row> row) { int capacity = 4 * 1024; WritableColumnVector[] columnVectors; if (memMode == MemoryMode.OFF_HEAP) { columnVectors = OffHeapColumnVector.allocateColumns(capacity, schema); } else { columnVectors = OnHeapColumnVector.allocateColumns(capacity, schema); } int n = 0; while (row.hasNext()) { Row r = row.next(); for (int i = 0; i < schema.fields().length; i++) { appendValue(columnVectors[i], schema.fields()[i].dataType(), r, i); } n++; } ColumnarBatch batch = new ColumnarBatch(columnVectors); batch.setNumRows(n); return batch; } }
/** * Converts an iterator of rows into a single ColumnBatch. */ public static ColumnarBatch toBatch( StructType schema, MemoryMode memMode, Iterator<Row> row) { int capacity = 4 * 1024; WritableColumnVector[] columnVectors; if (memMode == MemoryMode.OFF_HEAP) { columnVectors = OffHeapColumnVector.allocateColumns(capacity, schema); } else { columnVectors = OnHeapColumnVector.allocateColumns(capacity, schema); } int n = 0; while (row.hasNext()) { Row r = row.next(); for (int i = 0; i < schema.fields().length; i++) { appendValue(columnVectors[i], schema.fields()[i].dataType(), r, i); } n++; } ColumnarBatch batch = new ColumnarBatch(columnVectors); batch.setNumRows(n); return batch; } }
columnVectors = OffHeapColumnVector.allocateColumns(capacity, batchSchema); } else { columnVectors = OnHeapColumnVector.allocateColumns(capacity, batchSchema);
columnVectors = OffHeapColumnVector.allocateColumns(capacity, batchSchema); } else { columnVectors = OnHeapColumnVector.allocateColumns(capacity, batchSchema);
columnVectors = OffHeapColumnVector.allocateColumns(capacity, resultSchema); } else { columnVectors = OnHeapColumnVector.allocateColumns(capacity, resultSchema);
columnVectors = OffHeapColumnVector.allocateColumns(capacity, resultSchema); } else { columnVectors = OnHeapColumnVector.allocateColumns(capacity, resultSchema);