@Override public void transform(final INPUT inputValue, Emitter<Tuple2<JOIN_KEY, INPUT>> emitter) throws Exception { emitter.emit(new Tuple2<>(joiner.joinOn(inputStageName, inputValue), inputValue)); } }
@Override public void transform(Tuple2<JOIN_KEY, List<JoinElement<INPUT>>> input, Emitter<OUT> emitter) throws Exception { emitter.emit(joiner.merge(input._1(), input._2())); } }
@Override public Iterable<OUT> call(Tuple2<JOIN_KEY, List<JoinElement<INPUT_RECORD>>> input) throws Exception { if (joinFunction == null) { BatchJoiner<JOIN_KEY, INPUT_RECORD, OUT> joiner = pluginFunctionContext.createPlugin(); BatchJoinerRuntimeContext context = pluginFunctionContext.createBatchRuntimeContext(); joiner.initialize(context); joinFunction = new TrackedTransform<>(new JoinOnTransform<>(joiner), pluginFunctionContext.createStageMetrics(), Constants.Metrics.JOIN_KEYS, Constants.Metrics.RECORDS_OUT, pluginFunctionContext.getDataTracer(), pluginFunctionContext.getStageStatisticsCollector()); emitter = new DefaultEmitter<>(); } emitter.reset(); joinFunction.transform(input, emitter); return emitter.getEntries(); }
joiner.initialize(joinerRuntimeContext); for (final String inputStageName : joiner.getJoinConfig().getRequiredInputs()) { SparkPairCollection<Object, Object> preJoinCollection = preJoinStreams.get(inputStageName);
@Override public Iterable<Tuple2<JOIN_KEY, INPUT_RECORD>> call(INPUT_RECORD input) throws Exception { if (joinFunction == null) { BatchJoiner<JOIN_KEY, INPUT_RECORD, Object> joiner = pluginFunctionContext.createPlugin(); BatchJoinerRuntimeContext context = pluginFunctionContext.createBatchRuntimeContext(); joiner.initialize(context); joinFunction = new TrackedTransform<>(new JoinOnTransform<>(joiner, inputStageName), pluginFunctionContext.createStageMetrics(), Constants.Metrics.RECORDS_IN, null, pluginFunctionContext.getDataTracer(), pluginFunctionContext.getStageStatisticsCollector()); emitter = new DefaultEmitter<>(); } emitter.reset(); joinFunction.transform(input, emitter); return emitter.getEntries(); }
BatchJoiner<?, ?, ?> batchJoiner = pluginInstantiator.newPluginInstance(stageName, macroEvaluator); BatchJoinerRuntimeContext runtimeContext = createRuntimeContext(stageSpec); batchJoiner.initialize(runtimeContext); if (isMapPhase) { return getTrackedEmitKeyStep(