@Override public void destroy() { transformRunner.destroy(); } }
@Override protected void reduce(Object key, Iterable values, Context context) throws IOException, InterruptedException { try { transformRunner.transform(key, values.iterator()); } catch (StageFailureException e) { PIPELINE_LOG.error("{}", e.getMessage(), e.getCause()); Throwables.propagate(e.getCause()); } catch (Exception e) { Throwables.propagate(e); } }
@Override public void initialize(MapReduceTaskContext<Object, Object> context) throws Exception { // get source, transform, sink ids from program properties Map<String, String> properties = context.getSpecification().getProperties(); if (Boolean.valueOf(properties.get(Constants.STAGE_LOGGING_ENABLED))) { LogStageInjector.start(); } transformRunner = new TransformRunner<>(context, reducerMetrics); }
private OutputWriter<Object, Object> getSinkWriter(MapReduceTaskContext<Object, Object> context, PipelinePhase pipelinePhase, Configuration hConf) { Set<StageSpec> reducers = pipelinePhase.getStagesOfType(BatchAggregator.PLUGIN_TYPE, BatchJoiner.PLUGIN_TYPE); JobContext hadoopContext = context.getHadoopContext(); if (!reducers.isEmpty() && hadoopContext instanceof Mapper.Context) { return new SingleOutputWriter<>(context); } String sinkOutputsStr = hConf.get(ETLMapReduce.SINK_OUTPUTS_KEY); // should never happen, this is set in initialize Preconditions.checkNotNull(sinkOutputsStr, "Sink outputs not found in Hadoop conf."); Map<String, SinkOutput> sinkOutputs = GSON.fromJson(sinkOutputsStr, ETLMapReduce.SINK_OUTPUTS_TYPE); return hasSingleOutput(sinkOutputs) ? new SingleOutputWriter<>(context) : new MultiOutputWriter<>(context, sinkOutputs); }
this.outputWriter = getSinkWriter(context, phaseSpec.getPhase(), hConf);
@Override public void map(Object key, Object value, Mapper.Context context) throws IOException, InterruptedException { try { transformRunner.transform(key, value); } catch (StageFailureException e) { PIPELINE_LOG.error("{}", e.getMessage(), e.getCause()); Throwables.propagate(e.getCause()); } catch (Exception e) { Throwables.propagate(e); } }
@Override public void initialize(MapReduceTaskContext<Object, Object> context) throws Exception { // get source, transform, sink ids from program properties Map<String, String> properties = context.getSpecification().getProperties(); if (Boolean.valueOf(properties.get(Constants.STAGE_LOGGING_ENABLED))) { LogStageInjector.start(); } transformRunner = new TransformRunner<>(context, mapperMetrics); }
@Override public void destroy() { transformRunner.destroy(); } }