@Override public void emit(Object value) { RecordInfo record = getPipeRecord(value); for (PipeStage<RecordInfo> outputConsumer : outputConsumers) { outputConsumer.consume(record); } }
/** * Get a builder to create a PipeEmitter for the specified stage * * @param stageName the stage name * @return a builder to create a PipeEmitter for the specified stage */ public static Builder builder(String stageName) { return new Builder(stageName); }
@Override public StructuredRecord fromWritable(StructuredRecordWritable val) { return val.get(); } });
@Override protected void configure() { setName(phaseSpec.getPhaseName()); setDescription("CustomAction phase executor. " + phaseSpec.getPhaseName()); // add source, sink, transform ids to the properties. These are needed at runtime to instantiate the plugins Map<String, String> properties = new HashMap<>(); properties.put(Constants.PIPELINEID, GSON.toJson(phaseSpec)); setProperties(properties); }
@Override protected void configure() { setName(phaseSpec.getPhaseName()); setDescription("Condition phase executor. " + phaseSpec.getPhaseName()); // add source, sink, transform ids to the properties. These are needed at runtime to instantiate the plugins Map<String, String> properties = new HashMap<>(); properties.put(Constants.PIPELINEID, GSON.toJson(phaseSpec)); setProperties(properties); }
@Override public void map(Object key, Object value, Mapper.Context context) throws IOException, InterruptedException { try { transformRunner.transform(key, value); } catch (StageFailureException e) { PIPELINE_LOG.error("{}", e.getMessage(), e.getCause()); Throwables.propagate(e.getCause()); } catch (Exception e) { Throwables.propagate(e); } }
/** * Instantiates and initializes the plugin for the stage. * * @param stageInfo the stage info * @return the initialized Transformation * @throws InstantiationException if the plugin for the stage could not be instantiated * @throws Exception if there was a problem initializing the plugin */ private <T extends Transformation & StageLifecycle<BatchRuntimeContext>> Transformation getInitializedTransformation(StageSpec stageInfo) throws Exception { BatchRuntimeContext runtimeContext = createRuntimeContext(stageInfo); T plugin = pluginInstantiator.newPluginInstance(stageInfo.getName(), macroEvaluator); plugin.initialize(runtimeContext); return plugin; }
@TransactionPolicy(TransactionControl.EXPLICIT) @Override public void destroy() { WorkflowContext workflowContext = getContext(); PipelineRuntime pipelineRuntime = new PipelineRuntime(workflowContext, workflowMetrics); if (workflowContext.getDataTracer(PostAction.PLUGIN_TYPE).isEnabled()) { return; } for (Map.Entry<String, PostAction> endingActionEntry : postActions.entrySet()) { String name = endingActionEntry.getKey(); PostAction action = endingActionEntry.getValue(); StageSpec stageSpec = postActionSpecs.get(name); BatchActionContext context = new WorkflowBackedActionContext(workflowContext, pipelineRuntime, stageSpec); try { action.run(context); } catch (Throwable t) { LOG.error("Error while running ending action {}.", name, t); } } } }
/** * Consume a record and send it to the relevant output transforms. * * @param input the record to consume * @throws StageFailureException if there was an exception consuming the input */ public void consume(T input) { try { consumeInput(input); } catch (StageFailureException e) { // Another stage has already failed, just throw the exception as-is throw e; } catch (Exception e) { Throwable rootCause = Throwables.getRootCause(e); // Create StageFailureException to save the Stage information throw new StageFailureException( String.format("Failed to execute pipeline stage '%s' with the error: %s. Please review your pipeline " + "configuration and check the system logs for more details.", stageName, rootCause.getMessage()), rootCause); } }
public void runOneIteration(IN input) { for (String stageName : startingPoints) { PipeStage<RecordInfo> pipeStage = pipeStages.get(stageName); pipeStage.consume(RecordInfo.builder(input, stageName, RecordType.OUTPUT).build()); } }
/** * Get a builder to create a PipeEmitter for the specified stage * * @param stageName the stage name * @return a builder to create a PipeEmitter for the specified stage */ public static Builder builder(String stageName) { return new Builder(stageName); }
@Override public PipeEmitter build() { return new ConnectorSourceEmitter(stageName, outputConsumers, outputPortConsumers, errorConsumers, alertConsumers); } }
@Override public StructuredRecordWritable toWritable(StructuredRecord val) { return new StructuredRecordWritable(val); }
public PipeEmitter build() { return new PipeEmitter(stageName, outputConsumers, outputPortConsumers, errorConsumers, alertConsumers); } }
@Override public void destroy() { for (PipeStage stage : pipeStages.values()) { stage.destroy(); } } }
public void transform(KEY key, VALUE value) throws Exception { KeyValue<KEY, VALUE> input = new KeyValue<>(key, value); transformExecutor.runOneIteration(input); }
@Override protected void reduce(Object key, Iterable values, Context context) throws IOException, InterruptedException { try { transformRunner.transform(key, values.iterator()); } catch (StageFailureException e) { PIPELINE_LOG.error("{}", e.getMessage(), e.getCause()); Throwables.propagate(e.getCause()); } catch (Exception e) { Throwables.propagate(e); } }
@Override public void emit(String port, Object value) { if (port == null) { throw new IllegalArgumentException("Port cannot be null."); } RecordInfo record = getPipeRecord(value); for (PipeStage<RecordInfo> outputPortConsumer : outputPortConsumers.get(port)) { outputPortConsumer.consume(record); } }
@Override public void emitAlert(Map<String, String> payload) { Alert alert = new Alert(stageName, ImmutableMap.copyOf(payload)); RecordInfo<Alert> alertRecord = RecordInfo.builder(alert, stageName, RecordType.ALERT).build(); for (PipeStage<RecordInfo<Alert>> alertConsumer : alertConsumers) { alertConsumer.consume(alertRecord); } }