public <T> T createPlugin() throws Exception { if (Constants.Connector.PLUGIN_TYPE.equals(stageSpec.getPluginType())) { String connectorType = stageSpec.getPlugin().getProperties().get(Constants.Connector.TYPE); // ok to pass in null to constructors here since we are only going to use the transform method if (connectorType.equals(Constants.Connector.SOURCE_TYPE)) { return (T) new SingleConnectorSource(null, null); } else { return (T) new SingleConnectorSink(null, null); } } MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(arguments, logicalStartTime, secureStore, namespace); return getPluginContext().newPluginInstance(stageSpec.getName(), macroEvaluator); }
public String getStageName() { return stageSpec.getName(); }
PluginSpec pluginSpec = stageSpec.getPlugin(); PluginProperties pluginProperties = PluginProperties.builder().addAll(pluginSpec.getProperties()).build(); properties.put(STAGE_NAME, stageSpec.getName()); properties.put(Constants.PIPELINEID, GSON.toJson(phaseSpec, BatchPhaseSpec.class)); setProperties(properties);
String stageName = spec.getName(); resolvedProperties.put(stageName, workflowContext.getPluginProperties(stageName, macroEvaluator).getProperties()); allStageOperations.put(stageSpec.getName(), new ArrayList<>()); if (BatchJoiner.PLUGIN_TYPE.equals(stageSpec.getPlugin().getType())) { noMergeRequiredStages.add(stageSpec.getName()); for (StageSpec stageSpec : stageSpecs.values()) { StageOperationsValidator.Builder builder = new StageOperationsValidator.Builder(allStageOperations.get(stageSpec.getName())); Map<String, Schema> inputSchemas = stageSpec.getInputSchemas(); if (BatchJoiner.PLUGIN_TYPE.equals(stageSpec.getPlugin().getType())) { for (Map.Entry<String, Schema> entry : inputSchemas.entrySet()) { if (entry.getValue().getFields() != null) { Schema outputSchema = stageSpec.getOutputSchema(); if (outputSchema != null && outputSchema.getFields() != null) { stageOutputs.addAll(outputSchema.getFields().stream().map(Schema.Field::getName) StageOperationsValidator stageOperationsValidator = builder.build(); stageOperationsValidator.validate(); LOG.trace("Stage Name: {}", stageSpec.getName()); LOG.trace("Stage Operations {}", GSON.toJson(allStageOperations.get(stageSpec.getName()))); LOG.trace("Stage inputs: {}", stageInputs); LOG.trace("Stage outputs: {}", stageOutputs); InvalidFieldOperations invalidFieldOperations = stageOperationsValidator.getStageInvalids();
private PipeStage getPipeStage(PipelinePhase pipeline, String stageName, Map<String, PipeStage> pipeStages, OutputWriter<?, ?> outputWriter) throws Exception { StageSpec stageSpec = pipeline.getStage(stageName); String pluginType = stageSpec.getPluginType(); ConnectorSourceEmitter.builder(stageName) : PipeEmitter.builder(stageName); Map<String, StageSpec.Port> outputPorts = stageSpec.getOutputPorts(); for (String outputStageName : pipeline.getDag().getNodeOutputs(stageName)) { StageSpec outputStageSpec = pipeline.getStage(outputStageName); String outputStageType = outputStageSpec.getPluginType(); PipeStage outputPipeStage = pipeStages.get(outputStageName); outputStageSpec.getPlugin().getProperties().get(Constants.Connector.ORIGINAL_NAME) : outputStageName;
StageSpec stageSpec = pipelinePhase.getStage(stageName); String pluginType = stageSpec.getPluginType(); Set<String> outputs = pipelinePhase.getStageOutputs(stageSpec.getName()); for (String output : outputs) { String outputPluginType = pipelinePhase.getStage(output).getPluginType(); if (!Constants.Connector.PLUGIN_TYPE.equals(inputStageSpec.getPluginType()) && !Constants.Connector.PLUGIN_TYPE.equals(pluginType)) { port = inputStageSpec.getOutputPorts().get(stageName).getPort();
String stageName = actionSpec.getName(); postActions.put(stageName, pluginContext.newPluginInstance(stageName, macroEvaluator)); stageSpecs.put(stageName, StageSpec.builder(stageName, actionSpec.getPluginSpec()) .setStageLoggingEnabled(spec.isStageLoggingEnabled()) .setProcessTimingEnabled(spec.isProcessTimingEnabled()) String stageName = stageSpec.getName(); stageSpecs.put(stageName, stageSpec); if (AlertPublisher.PLUGIN_TYPE.equals(stageSpec.getPluginType())) { AlertPublisher alertPublisher = context.newPluginInstance(stageName, macroEvaluator); alertPublishers.put(stageName, alertPublisher);
@Override public Void call(JavaRDD<Alert> data, Time batchTime) throws Exception { MacroEvaluator evaluator = new DefaultMacroEvaluator(new BasicArguments(sec), batchTime.milliseconds(), sec.getSecureStore(), sec.getNamespace()); PluginContext pluginContext = new SparkPipelinePluginContext(sec.getPluginContext(), sec.getMetrics(), stageSpec.isStageLoggingEnabled(), stageSpec.isProcessTimingEnabled()); String stageName = stageSpec.getName(); AlertPublisher alertPublisher = pluginContext.newPluginInstance(stageName, evaluator); PipelineRuntime pipelineRuntime = new SparkPipelineRuntime(sec, batchTime.milliseconds()); AlertPublisherContext alertPublisherContext = new DefaultAlertPublisherContext(pipelineRuntime, stageSpec, sec.getMessagingContext(), sec.getAdmin()); alertPublisher.initialize(alertPublisherContext); StageMetrics stageMetrics = new DefaultStageMetrics(sec.getMetrics(), stageName); TrackedIterator<Alert> trackedAlerts = new TrackedIterator<>(data.collect().iterator(), stageMetrics, Constants.Metrics.RECORDS_IN); alertPublisher.publish(trackedAlerts); alertPublisher.destroy(); return null; } }
if (reducers.size() > 1) { Iterator<StageSpec> reducerIter = reducers.iterator(); StringBuilder reducersStr = new StringBuilder(reducerIter.next().getName()); while (reducerIter.hasNext()) { reducersStr.append(","); reducersStr.append(reducerIter.next().getName()); String pluginType = stageSpec.getPluginType(); boolean isConnectorSource = Constants.Connector.PLUGIN_TYPE.equals(pluginType) && phase.getSources().contains(stageName);
int sourceCount = 0; for (StageSpec stageSpec : spec.getStages()) { if (RealtimeSource.PLUGIN_TYPE.equals(stageSpec.getPlugin().getType())) { sourceCount++;
boolean hasErrors, boolean hasAlerts) { if (hasErrors || hasAlerts || stageSpec.getOutputPorts().size() > 1) { if (SplitterTransform.PLUGIN_TYPE.equals(stageSpec.getPluginType())) { for (StageSpec.Port portSpec : stageSpec.getOutputPorts().values()) { String port = portSpec.getPort(); SparkCollection<Object> portData = stageData.flatMap(stageSpec, Compat.convert(new OutputPassFilter<>(port)));
String pluginType = stageSpec.getPluginType(); boolean isConnectorSource = Constants.Connector.PLUGIN_TYPE.equals(pluginType) && phase.getSources().contains(stageName);
@TransactionPolicy(TransactionControl.EXPLICIT) @Override public void initialize(WorkflowContext context) throws Exception { super.initialize(context); postActions = new LinkedHashMap<>(); BatchPipelineSpec batchPipelineSpec = GSON.fromJson(context.getWorkflowSpecification().getProperty("pipeline.spec"), BatchPipelineSpec.class); MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(new BasicArguments(context.getToken(), context.getRuntimeArguments()), context.getLogicalStartTime(), context, context.getNamespace()); postActionSpecs = new HashMap<>(); for (ActionSpec actionSpec : batchPipelineSpec.getEndingActions()) { String name = actionSpec.getName(); postActions.put(name, (PostAction) context.newPluginInstance(name, macroEvaluator)); postActionSpecs.put(name, StageSpec.builder(name, actionSpec.getPluginSpec()) .setProcessTimingEnabled(batchPipelineSpec.isProcessTimingEnabled()) .setStageLoggingEnabled(batchPipelineSpec.isStageLoggingEnabled()) .build()); } }
public StageMetrics createStageMetrics() { return new DefaultStageMetrics(metrics, stageSpec.getName()); }
useSpark = config.getEngine() == Engine.SPARK; for (StageSpec stageSpec : spec.getStages()) { stageSpecs.put(stageSpec.getName(), stageSpec); String pluginType = stageSpec.getPlugin().getType(); if (SparkCompute.PLUGIN_TYPE.equals(pluginType) || SparkSink.PLUGIN_TYPE.equals(pluginType)) { useSpark = true;
sec.getNamespace()); PluginContext pluginContext = new SparkPipelinePluginContext(sec.getPluginContext(), sec.getMetrics(), stageSpec.isStageLoggingEnabled(), stageSpec.isProcessTimingEnabled()); final SparkBatchSinkFactory sinkFactory = new SparkBatchSinkFactory(); final String stageName = stageSpec.getName(); final BatchSink<Object, Object, Object> batchSink = pluginContext.newPluginInstance(stageName, evaluator); final PipelineRuntime pipelineRuntime = new SparkPipelineRuntime(sec, logicalStartTime);
String stageName = stageSpec.getName(); String pluginType = stageSpec.getPluginType(); StageMetrics stageMetrics = new DefaultStageMetrics(metrics, stageName); TaskAttemptContext taskAttemptContext = (TaskAttemptContext) taskContext.getHadoopContext();
if (BatchSource.PLUGIN_TYPE.equals(stageSpec.getPlugin().getType())) { sourceCount++;
@Override public void run() { JavaPairRDD<Object, Object> sinkRDD = rdd.flatMapToPair(sinkFunction); sinkFactory.writeFromRDD(sinkRDD, sec, stageSpec.getName(), Object.class, Object.class); } };