BasicWorkflowContext(WorkflowSpecification workflowSpec, WorkflowToken token, Program program, ProgramOptions programOptions, CConfiguration cConf, MetricsCollectionService metricsCollectionService, DatasetFramework datasetFramework, TransactionSystemClient txClient, DiscoveryServiceClient discoveryServiceClient, Map<String, WorkflowNodeState> nodeStates, @Nullable PluginInstantiator pluginInstantiator, SecureStore secureStore, SecureStoreManager secureStoreManager, MessagingService messagingService, @Nullable ConditionSpecification conditionSpecification, MetadataReader metadataReader, MetadataPublisher metadataPublisher) { super(program, programOptions, cConf, new HashSet(), datasetFramework, txClient, discoveryServiceClient, false, metricsCollectionService, Collections.singletonMap(Constants.Metrics.Tag.WORKFLOW_RUN_ID, ProgramRunners.getRunId(programOptions).getId()), secureStore, secureStoreManager, messagingService, pluginInstantiator, metadataReader, metadataPublisher); this.workflowSpec = workflowSpec; this.conditionSpecification = conditionSpecification; this.token = token; this.nodeStates = nodeStates; }
BasicWorkflowContext(WorkflowSpecification workflowSpec, WorkflowToken token, Program program, ProgramOptions programOptions, CConfiguration cConf, MetricsCollectionService metricsCollectionService, DatasetFramework datasetFramework, TransactionSystemClient txClient, DiscoveryServiceClient discoveryServiceClient, Map<String, WorkflowNodeState> nodeStates, @Nullable PluginInstantiator pluginInstantiator, SecureStore secureStore, SecureStoreManager secureStoreManager, MessagingService messagingService, @Nullable ConditionSpecification conditionSpecification, MetadataReader metadataReader, MetadataPublisher metadataPublisher, NamespaceQueryAdmin namespaceQueryAdmin) { super(program, programOptions, cConf, new HashSet<>(), datasetFramework, txClient, discoveryServiceClient, false, metricsCollectionService, Collections.singletonMap(Constants.Metrics.Tag.WORKFLOW_RUN_ID, ProgramRunners.getRunId(programOptions).getId()), secureStore, secureStoreManager, messagingService, pluginInstantiator, metadataReader, metadataPublisher, namespaceQueryAdmin); this.workflowSpec = workflowSpec; this.conditionSpecification = conditionSpecification; this.token = token; this.nodeStates = nodeStates; }
FlowProgramController(Table<String, Integer, ProgramController> flowlets, Program program, ProgramOptions options, FlowSpecification flowSpec, Multimap<String, QueueName> consumerQueues) { super(program.getId().run(ProgramRunners.getRunId(options))); this.flowlets = flowlets; this.program = program; this.options = options; this.flowSpec = flowSpec; this.consumerQueues = consumerQueues; started(); }
InMemoryProgramController(Table<String, Integer, ProgramController> components, Program program, ProgramOptions options) { super(program.getId().run(ProgramRunners.getRunId(options))); this.program = program; this.components = components; this.options = options; this.liveComponents = new AtomicLong(components.size()); started(); monitorComponents(); }
InMemoryProgramController(Table<String, Integer, ProgramController> components, Program program, ProgramOptions options) { super(program.getId().run(ProgramRunners.getRunId(options))); this.program = program; this.components = components; this.options = options; this.liveComponents = new AtomicLong(components.size()); started(); monitorComponents(); }
RunId runId = ProgramRunners.getRunId(options); Table<String, Integer, ProgramController> components = HashBasedTable.create(); try {
RunId runId = ProgramRunners.getRunId(options); Table<String, Integer, ProgramController> components = HashBasedTable.create(); try {
/** * Set up Logging Context so the Log is tagged correctly for the Program. * Reset the context once done. */ private void logProgramStart(Program program, ProgramOptions options) { LoggingContext loggingContext = LoggingContextHelper.getLoggingContext(program.getNamespaceId(), program.getApplicationId(), program.getName(), program.getType(), ProgramRunners.getRunId(options).getId(), options.getArguments().asMap()); Cancellable saveContextCancellable = LoggingContextAccessor.setLoggingContext(loggingContext); String userArguments = Joiner.on(", ").withKeyValueSeparator("=").join(options.getUserArguments()); LOG.info("Starting {} Program '{}' with Arguments [{}]", program.getType(), program.getName(), userArguments); saveContextCancellable.cancel(); }
/** * Set up Logging Context so the Log is tagged correctly for the Program. * Reset the context once done. */ private void logProgramStart(Program program, ProgramOptions options) { LoggingContext loggingContext = LoggingContextHelper.getLoggingContext(program.getNamespaceId(), program.getApplicationId(), program.getName(), program.getType(), ProgramRunners.getRunId(options).getId(), options.getArguments().asMap()); Cancellable saveContextCancellable = LoggingContextAccessor.setLoggingContext(loggingContext); String userArguments = Joiner.on(", ").withKeyValueSeparator("=").join(options.getUserArguments()); LOG.info("Starting {} Program '{}' with Arguments [{}]", program.getType(), program.getName(), userArguments); saveContextCancellable.cancel(); }
ProgramRunners.getRunId(workflowProgramOptions).getId()); systemArgumentsMap.put(ProgramOptionConstants.WORKFLOW_NODE_ID, nodeId); systemArgumentsMap.put(ProgramOptionConstants.PROGRAM_NAME_IN_WORKFLOW, name);
ProgramRunners.getRunId(workflowProgramOptions).getId()); systemArgumentsMap.put(ProgramOptionConstants.WORKFLOW_NODE_ID, nodeId); systemArgumentsMap.put(ProgramOptionConstants.PROGRAM_NAME_IN_WORKFLOW, name);
Preconditions.checkNotNull(workflowSpec, "Missing WorkflowSpecification for %s", program.getName()); final RunId runId = ProgramRunners.getRunId(options);
Preconditions.checkNotNull(workflowSpec, "Missing WorkflowSpecification for %s", program.getName()); final RunId runId = ProgramRunners.getRunId(options);
Preconditions.checkArgument(instanceCount > 0, "Invalid or missing instance count"); RunId runId = ProgramRunners.getRunId(options);
LOG.info("Setting up for MapReduce job: namespaceId={}, applicationId={}, program={}, runid={}", programId.getNamespace(), programId.getApplication(), programId.getProgram(), ProgramRunners.getRunId(contextConfig.getProgramOptions()));
LOG.info("Setting up for MapReduce job: namespaceId={}, applicationId={}, program={}, runid={}", programId.getNamespace(), programId.getApplication(), programId.getProgram(), ProgramRunners.getRunId(contextConfig.getProgramOptions()));
private void runAndWait(ProgramRunner programRunner, Program program, ProgramOptions options) throws Exception { Closeable closeable = createCloseable(programRunner, program); // Publish the program's starting state RunId runId = ProgramRunners.getRunId(options); String twillRunId = options.getArguments().getOption(ProgramOptionConstants.TWILL_RUN_ID); ProgramDescriptor programDescriptor = new ProgramDescriptor(program.getId(), program.getApplicationSpecification()); programStateWriter.start(program.getId().run(runId), options, twillRunId, programDescriptor); ProgramController controller; try { controller = programRunner.run(program, options); } catch (Throwable t) { // If there is any exception when running the program, close the program to release resources. // Otherwise it will be released when the execution completed. programStateWriter.error(program.getId().run(runId), t); Closeables.closeQuietly(closeable); throw t; } blockForCompletion(closeable, controller); if (controller instanceof WorkflowDataProvider) { updateWorkflowToken(((WorkflowDataProvider) controller).getWorkflowToken()); } else { // This shouldn't happen throw new IllegalStateException("No WorkflowToken available after program completed: " + program.getId()); } }
private void runAndWait(ProgramRunner programRunner, Program program, ProgramOptions options) throws Exception { Closeable closeable = createCloseable(programRunner, program); // Publish the program's starting state RunId runId = ProgramRunners.getRunId(options); String twillRunId = options.getArguments().getOption(ProgramOptionConstants.TWILL_RUN_ID); ProgramDescriptor programDescriptor = new ProgramDescriptor(program.getId(), program.getApplicationSpecification()); programStateWriter.start(program.getId().run(runId), options, twillRunId, programDescriptor); ProgramController controller; try { controller = programRunner.run(program, options); } catch (Throwable t) { // If there is any exception when running the program, close the program to release resources. // Otherwise it will be released when the execution completed. programStateWriter.error(program.getId().run(runId), t); Closeables.closeQuietly(closeable); throw t; } blockForCompletion(closeable, controller); if (controller instanceof WorkflowDataProvider) { updateWorkflowToken(((WorkflowDataProvider) controller).getWorkflowToken()); } else { // This shouldn't happen throw new IllegalStateException("No WorkflowToken available after program completed: " + program.getId()); } }
/** * Creates logging context for MapReduce program. If the program is started * by Workflow an instance of {@link WorkflowProgramLoggingContext} is returned, * otherwise an instance of {@link MapReduceLoggingContext} is returned. */ private LoggingContext createMapReduceLoggingContext() { MapReduceContextConfig contextConfig = new MapReduceContextConfig(parameters.getHConf()); ProgramId programId = contextConfig.getProgramId(); RunId runId = ProgramRunners.getRunId(contextConfig.getProgramOptions()); WorkflowProgramInfo workflowProgramInfo = contextConfig.getWorkflowProgramInfo(); if (workflowProgramInfo == null) { return new MapReduceLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } String workflowId = workflowProgramInfo.getName(); String workflowRunId = workflowProgramInfo.getRunId().getId(); return new WorkflowProgramLoggingContext(programId.getNamespace(), programId.getApplication(), workflowId, workflowRunId, ProgramType.MAPREDUCE, programId.getProgram(), runId.getId()); }
/** * Creates logging context for MapReduce program. If the program is started * by Workflow an instance of {@link WorkflowProgramLoggingContext} is returned, * otherwise an instance of {@link MapReduceLoggingContext} is returned. */ private LoggingContext createMapReduceLoggingContext() { MapReduceContextConfig contextConfig = new MapReduceContextConfig(parameters.getHConf()); ProgramId programId = contextConfig.getProgramId(); RunId runId = ProgramRunners.getRunId(contextConfig.getProgramOptions()); WorkflowProgramInfo workflowProgramInfo = contextConfig.getWorkflowProgramInfo(); if (workflowProgramInfo == null) { return new MapReduceLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } String workflowId = workflowProgramInfo.getName(); String workflowRunId = workflowProgramInfo.getRunId().getId(); return new WorkflowProgramLoggingContext(programId.getNamespace(), programId.getApplication(), workflowId, workflowRunId, ProgramType.MAPREDUCE, programId.getProgram(), runId.getId()); }