@Override protected void initialize() throws Exception { SparkClientContext context = getContext(); String stageName = context.getSpecification().getProperty(STAGE_NAME); Class<?> externalProgramClass = context.loadPluginClass(stageName); // If the external program implements Spark, instantiate it and call initialize() to provide full lifecycle support if (Spark.class.isAssignableFrom(externalProgramClass)) { MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(new BasicArguments(context), context.getLogicalStartTime(), context, context.getNamespace()); delegateSpark = context.newPluginInstance(stageName, macroEvaluator); if (delegateSpark instanceof AbstractSpark) { //noinspection unchecked ((AbstractSpark) delegateSpark).initialize(context); } } }
@Override public void setSparkConf(SparkConf sparkConf) { if (sparkContext == null) { throw new UnsupportedOperationException("Spark configurations cannot be updated in realtime pipelines."); } sparkContext.setSparkConf(sparkConf); } }
public SparkPipelineRuntime(SparkClientContext context) { super(context.getNamespace(), context.getApplicationSpecification().getName(), context.getLogicalStartTime(), new BasicArguments(context), context.getMetrics(), context, context, context, context, context); }
context.setSparkConf(sparkConf); Map<String, String> properties = context.getSpecification().getProperties(); BatchPhaseSpec phaseSpec = GSON.fromJson(properties.get(Constants.PIPELINEID), BatchPhaseSpec.class); context.getLogicalStartTime(), context, context.getNamespace()); final SparkBatchSourceFactory sourceFactory = new SparkBatchSourceFactory(); final SparkBatchSinkFactory sinkFactory = new SparkBatchSinkFactory(); final Map<String, Integer> stagePartitions = new HashMap<>(); PluginContext pluginContext = new SparkPipelinePluginContext(context, context.getMetrics(), phaseSpec.isStageLoggingEnabled(), phaseSpec.isProcessTimingEnabled()); PipelinePluginInstantiator pluginInstantiator = new PipelinePluginInstantiator(pluginContext, context.getMetrics(), phaseSpec, new SingleConnectorFactory()); final PipelineRuntime pipelineRuntime = new PipelineRuntime(context); final Admin admin = context.getAdmin(); context.localize("HydratorSpark.config", configFile.toURI()); WorkflowToken token = context.getWorkflowToken(); if (token != null) { for (Map.Entry<String, String> entry : pipelineRuntime.getArguments().getAddedArguments().entrySet()) {
public void initialize() throws Exception { SparkClientContext context = getContext(); String arguments = Joiner.on(", ").withKeyValueSeparator("=").join(context.getRuntimeArguments()); WRAPPERLOGGER.info("Pipeline '{}' is started by user '{}' with arguments {}", context.getApplicationSpecification().getName(), UserGroupInformation.getCurrentUser().getShortUserName(), arguments); DataStreamsPipelineSpec spec = GSON.fromJson(context.getSpecification().getProperty(Constants.PIPELINEID), DataStreamsPipelineSpec.class); PipelinePluginContext pluginContext = new SparkPipelinePluginContext(context, context.getMetrics(), true, true); int numSources = 0; sparkConf.setMaster(String.format("local[%d]", numSources + 1)); context.setSparkConf(sparkConf); FileSet checkpointFileSet = context.getDataset(DataStreamsApp.CHECKPOINT_FILESET); String pipelineName = context.getApplicationSpecification().getName(); String checkpointDir = spec.getCheckpointDirectory(); Location pipelineCheckpointBase = checkpointFileSet.getBaseLocation().append(pipelineName); WRAPPERLOGGER.info("Pipeline '{}' running", context.getApplicationSpecification().getName());
@Override public void initialize() throws Exception { SparkClientContext context = getContext(); Map<String, String> args = context.getRuntimeArguments(); String localFilePath = args.get(LOCAL_FILE_RUNTIME_ARG); Preconditions.checkArgument(localFilePath != null, "Runtime argument %s must be set.", LOCAL_FILE_RUNTIME_ARG); context.localize(LOCAL_FILE_ALIAS, URI.create(localFilePath)); context.localize(LOCAL_ARCHIVE_ALIAS, createTemporaryArchiveFile(), true); } }
@Override public void initialize() throws Exception { SparkClientContext context = getContext(); context.setSparkConf(new SparkConf().set("spark.io.compression.codec", "org.apache.spark.io.LZFCompressionCodec")); Table totals = context.getDataset("totals"); totals.get(new Get("total").add("total")).getLong("total"); totals.put(new Put("total").add("total", 0L)); }
@TransactionPolicy(TransactionControl.EXPLICIT) @Override public void destroy() { super.destroy(); ProgramStatus status = getContext().getState().getStatus(); WRAPPERLOGGER.info("Pipeline '{}' {}", getContext().getApplicationSpecification().getName(), status == ProgramStatus.COMPLETED ? "succeeded" : status.name().toLowerCase()); }
@Override @TransactionPolicy(TransactionControl.EXPLICIT) public void destroy() { if (finisher != null) { finisher.onFinish(getContext().getState().getStatus() == ProgramStatus.COMPLETED); } for (File file : cleanupFiles) { if (!file.delete()) { LOG.warn("Failed to clean up resource {} ", file); } } } }
@Override public void initialize() throws Exception { SparkClientContext context = getContext(); Map<String, String> args = context.getRuntimeArguments(); String localFilePath = args.get(LOCAL_FILE_RUNTIME_ARG); Preconditions.checkArgument(localFilePath != null, "Runtime argument %s must be set.", LOCAL_FILE_RUNTIME_ARG); context.localize(LOCAL_FILE_ALIAS, URI.create(localFilePath)); context.localize(LOCAL_ARCHIVE_ALIAS, createTemporaryArchiveFile(), true); } }