protected static BufferAggregator[] makeBufferAggregators(Cursor cursor, List<AggregatorFactory> aggregatorSpecs) { BufferAggregator[] aggregators = new BufferAggregator[aggregatorSpecs.size()]; int aggregatorIndex = 0; for (AggregatorFactory spec : aggregatorSpecs) { aggregators[aggregatorIndex] = spec.factorizeBuffered(cursor.getColumnSelectorFactory()); ++aggregatorIndex; } return aggregators; }
@Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { return new SuppressedBufferAggregator(delegate.factorizeBuffered(metricFactory)); }
public BufferArrayGrouper( // the buffer returned from the below supplier can have dirty bits and should be cleared during initialization final Supplier<ByteBuffer> bufferSupplier, final ColumnSelectorFactory columnSelectorFactory, final AggregatorFactory[] aggregatorFactories, final int cardinality ) { Preconditions.checkNotNull(aggregatorFactories, "aggregatorFactories"); Preconditions.checkArgument(cardinality > 0, "Cardinality must a non-zero positive number"); this.bufferSupplier = Preconditions.checkNotNull(bufferSupplier, "bufferSupplier"); this.aggregators = new BufferAggregator[aggregatorFactories.length]; this.aggregatorOffsets = new int[aggregatorFactories.length]; this.cardinalityWithMissingValue = cardinality + 1; int offset = 0; for (int i = 0; i < aggregatorFactories.length; i++) { aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls(); } recordSize = offset; }
StreamingMergeSortedGrouper( final Supplier<ByteBuffer> bufferSupplier, final KeySerde<KeyType> keySerde, final ColumnSelectorFactory columnSelectorFactory, final AggregatorFactory[] aggregatorFactories, final long queryTimeoutAtMs ) { this.bufferSupplier = bufferSupplier; this.keySerde = keySerde; this.aggregators = new BufferAggregator[aggregatorFactories.length]; this.aggregatorOffsets = new int[aggregatorFactories.length]; this.keySize = keySerde.keySize(); int offset = keySize; for (int i = 0; i < aggregatorFactories.length; i++) { aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls(); } this.recordSize = offset; // queryTimeoutAtMs comes from System.currentTimeMillis(), but we should use System.nanoTime() to check timeout in // this class. See increaseWriteIndex() and increaseReadIndex(). this.hasQueryTimeout = queryTimeoutAtMs != QueryContexts.NO_TIMEOUT; final long timeoutNs = hasQueryTimeout ? TimeUnit.MILLISECONDS.toNanos(queryTimeoutAtMs - System.currentTimeMillis()) : QueryContexts.NO_TIMEOUT; this.queryTimeoutAtNs = System.nanoTime() + timeoutNs; }
public BufferHashGrouper( final Supplier<ByteBuffer> bufferSupplier, final KeySerde<KeyType> keySerde, final ColumnSelectorFactory columnSelectorFactory, final AggregatorFactory[] aggregatorFactories, final int bufferGrouperMaxSize, final float maxLoadFactor, final int initialBuckets, final boolean useDefaultSorting ) { super(bufferSupplier, keySerde, aggregatorFactories, bufferGrouperMaxSize); this.aggregatorFactories = aggregatorFactories; this.maxLoadFactor = maxLoadFactor > 0 ? maxLoadFactor : DEFAULT_MAX_LOAD_FACTOR; this.initialBuckets = initialBuckets > 0 ? Math.max(MIN_INITIAL_BUCKETS, initialBuckets) : DEFAULT_INITIAL_BUCKETS; if (this.maxLoadFactor >= 1.0f) { throw new IAE("Invalid maxLoadFactor[%f], must be < 1.0", maxLoadFactor); } int offset = HASH_SIZE + keySize; for (int i = 0; i < aggregatorFactories.length; i++) { aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls(); } this.bucketSize = offset; this.useDefaultSorting = useDefaultSorting; }
this.aggregatorFactories = aggregatorFactories; for (int i = 0; i < aggregatorFactories.length; i++) { aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls();
@Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory columnSelectorFactory) { final ValueMatcher valueMatcher = Filters.toFilter(filter).makeMatcher(columnSelectorFactory); return new FilteredBufferAggregator( valueMatcher, delegate.factorizeBuffered(columnSelectorFactory) ); }
for (int i = 0; i < aggregatorSpecs.size(); ++i) { AggregatorFactory aggregatorSpec = aggregatorSpecs.get(i); aggregators[i] = aggregatorSpec.factorizeBuffered(cursor.getColumnSelectorFactory()); metricNames[i] = aggregatorSpec.getName(); sizesRequired[i] = aggregatorSpec.getMaxIntermediateSizeWithNulls();
for (int i = 0; i < metrics.length; i++) { final AggregatorFactory agg = metrics[i]; getAggs()[i] = agg.factorizeBuffered(selectors.get(agg.getName()));
public <T> T[] runRelocateVerificationTest( AggregatorFactory factory, ColumnSelectorFactory selector, Class<T> clazz ) { T[] results = (T[]) Array.newInstance(clazz, 2); BufferAggregator agg = factory.factorizeBuffered(selector); ByteBuffer myBuf = ByteBuffer.allocate(10040902); agg.init(myBuf, 0); agg.aggregate(myBuf, 0); results[0] = (T) agg.get(myBuf, 0); byte[] theBytes = new byte[factory.getMaxIntermediateSizeWithNulls()]; myBuf.get(theBytes); ByteBuffer newBuf = ByteBuffer.allocate(941209); newBuf.position(7574); newBuf.put(theBytes); newBuf.position(0); agg.relocate(0, 7574, myBuf, newBuf); results[1] = (T) agg.get(newBuf, 7574); return results; }
@Test public void testStringLastBufferAggregator() { BufferAggregator agg = stringLastAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[stringLastAggFactory.getMaxIntermediateSize()]); agg.init(buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); Pair<Long, String> result = (Pair<Long, String>) agg.get(buffer, 0); Assert.assertEquals(strings[2], result.rhs); }
@Test public void testStringLastBufferAggregator() { BufferAggregator agg = stringLastAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[stringLastAggFactory.getMaxIntermediateSize()]); agg.init(buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); Pair<Long, String> result = (Pair<Long, String>) agg.get(buffer, 0); Assert.assertEquals(strings[0], result.rhs); }
@Test public void testStringLastCombiningBufferAggregator() { BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[stringLastAggFactory.getMaxIntermediateSize()]); agg.init(buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); Pair<Long, String> result = (Pair<Long, String>) agg.get(buffer, 0); Pair<Long, String> expected = pairs[2]; Assert.assertEquals(expected.lhs, result.lhs); Assert.assertEquals(expected.rhs, result.rhs); }
@Test public void testStringFirstCombiningBufferAggregator() { BufferAggregator agg = combiningAggFactory.factorizeBuffered( colSelectorFactory); ByteBuffer buffer = ByteBuffer.wrap(new byte[stringLastAggFactory.getMaxIntermediateSize()]); agg.init(buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); aggregate(agg, buffer, 0); Pair<Long, String> result = (Pair<Long, String>) agg.get(buffer, 0); Pair<Long, String> expected = pairs[3]; Assert.assertEquals(expected.lhs, result.lhs); Assert.assertEquals(expected.rhs, result.rhs); }
protected static BufferAggregator[] makeBufferAggregators(Cursor cursor, List<AggregatorFactory> aggregatorSpecs) { BufferAggregator[] aggregators = new BufferAggregator[aggregatorSpecs.size()]; int aggregatorIndex = 0; for (AggregatorFactory spec : aggregatorSpecs) { aggregators[aggregatorIndex] = spec.factorizeBuffered(cursor.getColumnSelectorFactory()); ++aggregatorIndex; } return aggregators; }
@Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { return new SuppressedBufferAggregator(delegate.factorizeBuffered(metricFactory)); }
public BufferArrayGrouper( // the buffer returned from the below supplier can have dirty bits and should be cleared during initialization final Supplier<ByteBuffer> bufferSupplier, final ColumnSelectorFactory columnSelectorFactory, final AggregatorFactory[] aggregatorFactories, final int cardinality ) { Preconditions.checkNotNull(aggregatorFactories, "aggregatorFactories"); Preconditions.checkArgument(cardinality > 0, "Cardinality must a non-zero positive number"); this.bufferSupplier = Preconditions.checkNotNull(bufferSupplier, "bufferSupplier"); this.aggregators = new BufferAggregator[aggregatorFactories.length]; this.aggregatorOffsets = new int[aggregatorFactories.length]; this.cardinalityWithMissingValue = cardinality + 1; int offset = 0; for (int i = 0; i < aggregatorFactories.length; i++) { aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls(); } recordSize = offset; }
StreamingMergeSortedGrouper( final Supplier<ByteBuffer> bufferSupplier, final KeySerde<KeyType> keySerde, final ColumnSelectorFactory columnSelectorFactory, final AggregatorFactory[] aggregatorFactories, final long queryTimeoutAtMs ) { this.bufferSupplier = bufferSupplier; this.keySerde = keySerde; this.aggregators = new BufferAggregator[aggregatorFactories.length]; this.aggregatorOffsets = new int[aggregatorFactories.length]; this.keySize = keySerde.keySize(); int offset = keySize; for (int i = 0; i < aggregatorFactories.length; i++) { aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls(); } this.recordSize = offset; // queryTimeoutAtMs comes from System.currentTimeMillis(), but we should use System.nanoTime() to check timeout in // this class. See increaseWriteIndex() and increaseReadIndex(). this.hasQueryTimeout = queryTimeoutAtMs != QueryContexts.NO_TIMEOUT; final long timeoutNs = hasQueryTimeout ? TimeUnit.MILLISECONDS.toNanos(queryTimeoutAtMs - System.currentTimeMillis()) : QueryContexts.NO_TIMEOUT; this.queryTimeoutAtNs = System.nanoTime() + timeoutNs; }
public BufferHashGrouper( final Supplier<ByteBuffer> bufferSupplier, final KeySerde<KeyType> keySerde, final ColumnSelectorFactory columnSelectorFactory, final AggregatorFactory[] aggregatorFactories, final int bufferGrouperMaxSize, final float maxLoadFactor, final int initialBuckets, final boolean useDefaultSorting ) { super(bufferSupplier, keySerde, aggregatorFactories, bufferGrouperMaxSize); this.aggregatorFactories = aggregatorFactories; this.maxLoadFactor = maxLoadFactor > 0 ? maxLoadFactor : DEFAULT_MAX_LOAD_FACTOR; this.initialBuckets = initialBuckets > 0 ? Math.max(MIN_INITIAL_BUCKETS, initialBuckets) : DEFAULT_INITIAL_BUCKETS; if (this.maxLoadFactor >= 1.0f) { throw new IAE("Invalid maxLoadFactor[%f], must be < 1.0", maxLoadFactor); } int offset = HASH_SIZE + keySize; for (int i = 0; i < aggregatorFactories.length; i++) { aggregators[i] = aggregatorFactories[i].factorizeBuffered(columnSelectorFactory); aggregatorOffsets[i] = offset; offset += aggregatorFactories[i].getMaxIntermediateSizeWithNulls(); } this.bucketSize = offset; this.useDefaultSorting = useDefaultSorting; }
@Override public BufferAggregator factorizeBuffered(ColumnSelectorFactory columnSelectorFactory) { final ValueMatcher valueMatcher = Filters.toFilter(filter).makeMatcher(columnSelectorFactory); return new FilteredBufferAggregator( valueMatcher, delegate.factorizeBuffered(columnSelectorFactory) ); }