@Override public void transform(Tuple2<GROUP_KEY, Iterable<GROUP_VAL>> input, Emitter<OUT_VAL> emitter) throws Exception { aggregator.aggregate(input._1(), input._2().iterator(), emitter); } }
@Override public void transform(final GROUP_VAL inputValue, Emitter<Tuple2<GROUP_KEY, GROUP_VAL>> emitter) throws Exception { keyEmitter.reset(); aggregator.groupBy(inputValue, keyEmitter); for (GROUP_KEY key : keyEmitter.getEntries()) { emitter.emit(new Tuple2<>(key, inputValue)); } } }
@Override public Iterable<Tuple2<GROUP_KEY, GROUP_VAL>> call(GROUP_VAL input) throws Exception { if (groupByFunction == null) { BatchAggregator<GROUP_KEY, GROUP_VAL, ?> aggregator = pluginFunctionContext.createPlugin(); aggregator.initialize(pluginFunctionContext.createBatchRuntimeContext()); groupByFunction = new TrackedTransform<>(new GroupByTransform<>(aggregator), pluginFunctionContext.createStageMetrics(), Constants.Metrics.RECORDS_IN, null, pluginFunctionContext.getDataTracer(), pluginFunctionContext.getStageStatisticsCollector()); emitter = new DefaultEmitter<>(); } emitter.reset(); groupByFunction.transform(input, emitter); return emitter.getEntries(); }
@Override public Iterable<RecordInfo<Object>> call(Tuple2<GROUP_KEY, Iterable<GROUP_VAL>> input) throws Exception { if (aggregateTransform == null) { BatchAggregator<GROUP_KEY, GROUP_VAL, OUT> aggregator = pluginFunctionContext.createPlugin(); aggregator.initialize(pluginFunctionContext.createBatchRuntimeContext()); aggregateTransform = new TrackedTransform<>(new AggregateTransform<>(aggregator), pluginFunctionContext.createStageMetrics(), Constants.Metrics.AGG_GROUPS, Constants.Metrics.RECORDS_OUT, pluginFunctionContext.getDataTracer(), pluginFunctionContext.getStageStatisticsCollector()); emitter = new CombinedEmitter<>(pluginFunctionContext.getStageName()); } emitter.reset(); aggregateTransform.transform(input, emitter); return emitter.getEmitted(); }
BatchAggregator<?, ?, ?> batchAggregator = pluginInstantiator.newPluginInstance(stageName, macroEvaluator); BatchRuntimeContext runtimeContext = createRuntimeContext(stageSpec); batchAggregator.initialize(runtimeContext); if (isMapPhase) { return getTrackedEmitKeyStep(new MapperAggregatorTransformation(batchAggregator, mapOutputKeyClassName,