private OperatorFactory createStreamingAggregationOperatorFactory() { return new StreamingAggregationOperatorFactory( 0, new PlanNodeId("test"), ImmutableList.of(VARCHAR), ImmutableList.of(VARCHAR), ImmutableList.of(0), AggregationNode.Step.SINGLE, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty()), LONG_SUM.bind(ImmutableList.of(1), Optional.empty())), new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig())); }
public PagesHashStrategyFactory compilePagesHashStrategyFactory(List<Type> types, List<Integer> joinChannels) { return compilePagesHashStrategyFactory(types, joinChannels, Optional.empty()); }
public LookupSourceSupplierFactory compileLookupSourceFactory(List<? extends Type> types, List<Integer> joinChannels, Optional<Integer> sortChannel) { return compileLookupSourceFactory(types, joinChannels, sortChannel, Optional.empty()); }
type(PagesHashStrategy.class)); FieldDefinition instanceSizeField = generateInstanceSize(classDefinition); FieldDefinition sizeField = classDefinition.declareField(a(PRIVATE, FINAL), "size", type(long.class)); List<FieldDefinition> channelFields = new ArrayList<>(); generateConstructor(classDefinition, joinChannels, sizeField, instanceSizeField, channelFields, joinChannelFields, hashChannelField); generateGetChannelCountMethod(classDefinition, outputChannels.size()); generateGetSizeInBytesMethod(classDefinition, sizeField); generateAppendToMethod(classDefinition, callSiteBinder, types, outputChannels, channelFields); generateHashPositionMethod(classDefinition, callSiteBinder, joinChannelTypes, joinChannelFields, hashChannelField); generateHashRowMethod(classDefinition, callSiteBinder, joinChannelTypes); generateRowEqualsRowMethod(classDefinition, callSiteBinder, joinChannelTypes); generatePositionEqualsRowMethod(classDefinition, callSiteBinder, joinChannelTypes, joinChannelFields, true); generatePositionEqualsRowMethod(classDefinition, callSiteBinder, joinChannelTypes, joinChannelFields, false); generatePositionNotDistinctFromRowWithPageMethod(classDefinition, callSiteBinder, joinChannelTypes, joinChannelFields); generatePositionEqualsRowWithPageMethod(classDefinition, callSiteBinder, joinChannelTypes, joinChannelFields); generatePositionEqualsPositionMethod(classDefinition, callSiteBinder, joinChannelTypes, joinChannelFields, true); generatePositionEqualsPositionMethod(classDefinition, callSiteBinder, joinChannelTypes, joinChannelFields, false); generateIsPositionNull(classDefinition, joinChannelFields); generateCompareSortChannelPositionsMethod(classDefinition, callSiteBinder, types, channelFields, sortChannel); generateIsSortChannelPositionNull(classDefinition, channelFields, sortChannel);
generateConstructor(classDefinition, joinChannels, sizeField, channelFields, joinChannelFields, hashChannelField); generateGetChannelCountMethod(classDefinition, channelFields); generateGetSizeInBytesMethod(classDefinition, sizeField); generateAppendToMethod(classDefinition, callSiteBinder, types, channelFields); generateHashPositionMethod(classDefinition, callSiteBinder, joinChannelTypes, joinChannelFields, hashChannelField); generateHashRowMethod(classDefinition, callSiteBinder, joinChannelTypes); generateRowEqualsRowMethod(classDefinition, callSiteBinder, joinChannelTypes); generatePositionEqualsRowMethod(classDefinition, callSiteBinder, joinChannelTypes, joinChannelFields); generatePositionEqualsRowWithPageMethod(classDefinition, callSiteBinder, joinChannelTypes, joinChannelFields); generatePositionEqualsPositionMethod(classDefinition, callSiteBinder, joinChannelTypes, joinChannelFields);
JoinCompiler joinCompiler = new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig()); List<Type> types = ImmutableList.of(VARCHAR, VARCHAR, BIGINT, DOUBLE, BOOLEAN, VARCHAR); List<Type> joinTypes = ImmutableList.of(VARCHAR, BIGINT, DOUBLE, BOOLEAN); PagesHashStrategyFactory pagesHashStrategyFactory = joinCompiler.compilePagesHashStrategyFactory(types, joinChannels, Optional.of(outputChannels)); PagesHashStrategy hashStrategy = pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel);
JoinCompiler joinCompiler = new JoinCompiler(); List<Type> types = ImmutableList.<Type>of(VARCHAR, VARCHAR, BIGINT, DOUBLE, BOOLEAN); List<Type> joinTypes = ImmutableList.<Type>of(VARCHAR, BIGINT, DOUBLE, BOOLEAN); PagesHashStrategyFactory pagesHashStrategyFactory = joinCompiler.compilePagesHashStrategyFactory(types, joinChannels); PagesHashStrategy hashStrategy = pagesHashStrategyFactory.createPagesHashStrategy(channels, hashChannel); PagesHashStrategy expectedHashStrategy = new SimplePagesHashStrategy(types, channels, joinChannels, hashChannel);
@BeforeMethod public void setUp() { executor = newCachedThreadPool(daemonThreadsNamed("test-executor-%s")); scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s")); driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION) .addPipelineContext(0, true, true, false) .addDriverContext(); operatorFactory = new StreamingAggregationOperatorFactory( 0, new PlanNodeId("test"), ImmutableList.of(BOOLEAN, VARCHAR, BIGINT), ImmutableList.of(VARCHAR), ImmutableList.of(1), AggregationNode.Step.SINGLE, ImmutableList.of(COUNT.bind(ImmutableList.of(0), Optional.empty()), LONG_SUM.bind(ImmutableList.of(2), Optional.empty())), new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig())); }
public StreamingAggregationOperator(OperatorContext operatorContext, List<Type> sourceTypes, List<Type> groupByTypes, List<Integer> groupByChannels, Step step, List<AccumulatorFactory> accumulatorFactories, JoinCompiler joinCompiler) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.systemMemoryContext = operatorContext.newLocalSystemMemoryContext(StreamingAggregationOperator.class.getSimpleName()); this.userMemoryContext = operatorContext.localUserMemoryContext(); this.groupByTypes = ImmutableList.copyOf(requireNonNull(groupByTypes, "groupByTypes is null")); this.groupByChannels = Ints.toArray(requireNonNull(groupByChannels, "groupByChannels is null")); this.accumulatorFactories = requireNonNull(accumulatorFactories, "accumulatorFactories is null"); this.step = requireNonNull(step, "step is null"); this.aggregates = setupAggregates(step, accumulatorFactories); this.pageBuilder = new PageBuilder(toTypes(groupByTypes, aggregates)); requireNonNull(joinCompiler, "joinCompiler is null"); requireNonNull(sourceTypes, "sourceTypes is null"); pagesHashStrategy = joinCompiler.compilePagesHashStrategyFactory(sourceTypes, groupByChannels, Optional.empty()) .createPagesHashStrategy( sourceTypes.stream() .map(type -> ImmutableList.<Block>of()) .collect(toImmutableList()), OptionalInt.empty()); }
public static class DefaultFactory implements Factory { private final OrderingCompiler orderingCompiler; private final JoinCompiler joinCompiler; private final boolean eagerCompact; private final FunctionRegistry functionRegistry; private final boolean groupByUsesEqualTo; @Inject public DefaultFactory(OrderingCompiler orderingCompiler, JoinCompiler joinCompiler, FeaturesConfig featuresConfig, Metadata metadata) { this.orderingCompiler = requireNonNull(orderingCompiler, "orderingCompiler is null"); this.joinCompiler = requireNonNull(joinCompiler, "joinCompiler is null"); this.eagerCompact = requireNonNull(featuresConfig, "featuresConfig is null").isPagesIndexEagerCompactionEnabled(); this.functionRegistry = requireNonNull(metadata, "metadata is null").getFunctionRegistry(); this.groupByUsesEqualTo = featuresConfig.isGroupByUsesEqualTo(); } @Override public PagesIndex newPagesIndex(List<Type> types, int expectedPositions) { return new PagesIndex(orderingCompiler, joinCompiler, functionRegistry, groupByUsesEqualTo, types, expectedPositions, eagerCompact); } }
private static GroupByHash createGroupByHash(List<Type> partitionTypes, List<Integer> partitionChannels, UpdateMemory updateMemory) { return GroupByHash.createGroupByHash( partitionTypes, Ints.toArray(partitionChannels), Optional.empty(), 1, false, new JoinCompiler(createTestMetadataManager(), new FeaturesConfig()), updateMemory); }
public PagesHashStrategy createPagesHashStrategy(List<Integer> joinChannels, OptionalInt hashChannel, Optional<List<Integer>> outputChannels) { try { return joinCompiler.compilePagesHashStrategyFactory(types, joinChannels, outputChannels) .createPagesHashStrategy(ImmutableList.copyOf(channels), hashChannel); } catch (Exception e) { log.error(e, "Lookup source compile failed for types=%s error=%s", types, e); } // if compilation fails, use interpreter return new SimplePagesHashStrategy( types, outputChannels.orElse(rangeList(types.size())), ImmutableList.copyOf(channels), joinChannels, hashChannel, Optional.empty(), functionRegistry, groupByUsesEqualTo); }
public LookupSource createLookupSource(List<Integer> joinChannels, Optional<Integer> hashChannel) { try { LookupSourceFactory lookupSourceFactory = joinCompiler.compileLookupSourceFactory(types, joinChannels); LookupSource lookupSource = lookupSourceFactory.createLookupSource( valueAddresses, ImmutableList.<List<Block>>copyOf(channels), hashChannel); return lookupSource; } catch (Exception e) { log.error(e, "Lookup source compile failed for types=%s error=%s", types, e); } // if compilation fails PagesHashStrategy hashStrategy = new SimplePagesHashStrategy( types, ImmutableList.<List<Block>>copyOf(channels), joinChannels, hashChannel); return new InMemoryJoinHash(valueAddresses, hashStrategy); }
@BeforeMethod public void setUp() { executor = newCachedThreadPool(daemonThreadsNamed("test-executor-%s")); scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s")); driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION) .addPipelineContext(0, true, true, false) .addDriverContext(); joinCompiler = new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig()); }
PagesHashStrategyFactory pagesHashStrategyFactory = joinCompiler.compilePagesHashStrategyFactory(this.types, outputChannels.build()); hashStrategy = pagesHashStrategyFactory.createPagesHashStrategy(this.channelBuilders, this.precomputedHashChannel);
LookupSourceFactory lookupSourceFactoryFactory = joinCompiler.compileLookupSourceFactory(types, Ints.asList(0));
@BeforeMethod public void setUp() { executor = newCachedThreadPool(daemonThreadsNamed("test-executor-%s")); scheduledExecutor = newScheduledThreadPool(2, daemonThreadsNamed("test-scheduledExecutor-%s")); driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION) .addPipelineContext(0, true, true, false) .addDriverContext(); joinCompiler = new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig()); }
PagesHashStrategyFactory pagesHashStrategyFactory = joinCompiler.compilePagesHashStrategyFactory(joinTypes, joinChannels);
private static JoinCompiler getJoinCompiler(boolean groupByUsesEqual) { return new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig().setGroupByUsesEqualTo(groupByUsesEqual)); }
public PagesHashStrategy createPagesHashStrategy(List<Integer> joinChannels, Optional<Integer> hashChannel) { try { return joinCompiler.compilePagesHashStrategyFactory(types, joinChannels) .createPagesHashStrategy(ImmutableList.copyOf(channels), hashChannel); } catch (Exception e) { log.error(e, "Lookup source compile failed for types=%s error=%s", types, e); } // if compilation fails, use interpreter return new SimplePagesHashStrategy(types, ImmutableList.<List<Block>>copyOf(channels), joinChannels, hashChannel); }