@Override public String apply(RunId input) { return input.getId(); } };
@Override public void process(WatchedEvent event) { if (event.getState() == Event.KeeperState.Expired) { LOG.warn("ZK Session expired for service {} with runId {}.", getServiceName(), runId.getId()); expired = true; } else if (event.getState() == Event.KeeperState.SyncConnected && expired) { LOG.info("Reconnected after expiration for service {} with runId {}", getServiceName(), runId.getId()); expired = false; logIfFailed(createLiveNode()); } } });
/** * @return time from the UUID if it is a time-based UUID, -1 otherwise. */ public static long getTime(RunId runId, TimeUnit timeUnit) { return getTime(runId.getId(), timeUnit); }
@Override public JsonElement serialize(RunId src, Type typeOfSrc, JsonSerializationContext context) { return context.serialize(src.getId()); } }
/** * Returns the zookeeper node path for the ephemeral instance node for this runId. */ protected final String getInstancePath() { return String.format("%s/%s", Constants.INSTANCES_PATH_PREFIX, getRunId().getId()); }
@Override public void error(Throwable cause) { Closeables.closeQuietly(closeable); nodeStates.put(nodeId, new WorkflowNodeState(nodeId, NodeStatus.FAILED, controller.getRunId().getId(), cause)); completion.setException(cause); } }, Threads.SAME_THREAD_EXECUTOR);
@Override public void error(Throwable cause) { Closeables.closeQuietly(closeable); nodeStates.put(nodeId, new WorkflowNodeState(nodeId, NodeStatus.FAILED, controller.getRunId().getId(), cause)); completion.setException(cause); } }, Threads.SAME_THREAD_EXECUTOR);
/** * Creates program arguments. It includes all configurations from the specification, excluding hConf and cConf. */ private Arguments createProgramArguments(TwillContext context, Map<String, String> configs) { Map<String, String> args = ImmutableMap.<String, String>builder() .put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(context.getInstanceId())) .put(ProgramOptionConstants.INSTANCES, Integer.toString(context.getInstanceCount())) .put(ProgramOptionConstants.RUN_ID, context.getApplicationRunId().getId()) .putAll(Maps.filterKeys(configs, Predicates.not(Predicates.in(ImmutableSet.of("hConf", "cConf"))))) .build(); return new BasicArguments(args); }
/** * Returns a set of extra system arguments that will be available through the {@link ProgramOptions#getArguments()} * for the program execution. */ protected Map<String, String> getExtraSystemArguments() { Map<String, String> args = new HashMap<>(); args.put(ProgramOptionConstants.INSTANCE_ID, context == null ? "0" : Integer.toString(context.getInstanceId())); args.put(ProgramOptionConstants.INSTANCES, context == null ? "1" : Integer.toString(context.getInstanceCount())); args.put(ProgramOptionConstants.TWILL_RUN_ID, context.getApplicationRunId().getId()); args.put(ProgramOptionConstants.HOST, context.getHost().getCanonicalHostName()); return args; }
@Override public void completed() { Closeables.closeQuietly(closeable); Set<Operation> fieldLineageOperations = new HashSet<>(); if (controller instanceof WorkflowDataProvider) { fieldLineageOperations.addAll(((WorkflowDataProvider) controller).getFieldLineageOperations()); } nodeStates.put(nodeId, new WorkflowNodeState(nodeId, NodeStatus.COMPLETED, fieldLineageOperations, controller.getRunId().getId(), null)); completion.set(null); }
@Override public void containerLaunched(String runnableName, int instanceId, String containerId) { super.containerLaunched(runnableName, instanceId, containerId); if (runningPublished.compareAndSet(false, true)) { // The program is marked as running when the first container for the program is launched programStateWriterWithHeartBeat.running(twillRunId.getId()); } }
/** * Updates the metrics tags based on the information in this class. */ public Map<String, String> updateMetricsTags(Map<String, String> tags) { tags.put(Constants.Metrics.Tag.WORKFLOW, getName()); tags.put(Constants.Metrics.Tag.WORKFLOW_RUN_ID, getRunId().getId()); tags.put(Constants.Metrics.Tag.NODE, getNodeId()); return tags; }
private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) { Map<String, String> systemOptions = Maps.newHashMap(); systemOptions.putAll(options.getArguments().asMap()); systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId)); systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances)); systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId()); systemOptions.put(ProgramOptionConstants.HOST, host); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments()); }
/** * Creates a new instance based on the given {@link WorkflowProgramInfo}. */ public static NameMappedDatasetFramework createFromWorkflowProgramInfo(DatasetFramework datasetFramework, WorkflowProgramInfo info, ApplicationSpecification appSpec) { Set<String> localDatasets = appSpec.getWorkflows().get(info.getName()).getLocalDatasetSpecs().keySet(); return new NameMappedDatasetFramework(datasetFramework, localDatasets, info.getRunId().getId()); }
private LoggingContext createLoggingContext(ProgramId programId, RunId runId, @Nullable WorkflowProgramInfo workflowProgramInfo) { if (workflowProgramInfo == null) { return new MapReduceLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } ProgramId workflowProramId = programId.getParent().workflow(workflowProgramInfo.getName()); return new WorkflowProgramLoggingContext(workflowProramId.getNamespace(), workflowProramId.getApplication(), workflowProramId.getProgram(), workflowProgramInfo.getRunId().getId(), ProgramType.MAPREDUCE, programId.getProgram(), runId.getId()); }
private String getJobName(BasicMapReduceContext context) { ProgramId programId = context.getProgram().getId(); // MRJobClient expects the following format (for RunId to be the first component) return String.format("%s.%s.%s.%s.%s", context.getRunId().getId(), ProgramType.MAPREDUCE.name().toLowerCase(), programId.getNamespace(), programId.getApplication(), programId.getProgram()); }
private String getServiceName() { ProgramId programId = getProgramRunId().getParent(); RunId runId = getRunId(); return String.format("workflow.%s.%s.%s.%s", programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } }
private String getServiceName() { ProgramId programId = getProgramRunId().getParent(); RunId runId = getRunId(); return String.format("workflow.%s.%s.%s.%s", programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } }
private LoggingContext createLoggingContext(ProgramId programId, RunId runId, @Nullable WorkflowProgramInfo workflowProgramInfo) { if (workflowProgramInfo == null) { return new SparkLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } ProgramId workflowProramId = Ids.namespace(programId.getNamespace()).app(programId.getApplication()) .workflow(workflowProgramInfo.getName()); return new WorkflowProgramLoggingContext(workflowProramId.getNamespace(), workflowProramId.getApplication(), workflowProramId.getProgram(), workflowProgramInfo.getRunId().getId(), ProgramType.SPARK, programId.getProgram(), runId.getId()); }
private LoggingContext createLoggingContext(ProgramId programId, RunId runId, @Nullable WorkflowProgramInfo workflowProgramInfo) { if (workflowProgramInfo == null) { return new SparkLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } ProgramId workflowProramId = Ids.namespace(programId.getNamespace()).app(programId.getApplication()) .workflow(workflowProgramInfo.getName()); return new WorkflowProgramLoggingContext(workflowProramId.getNamespace(), workflowProramId.getApplication(), workflowProramId.getProgram(), workflowProgramInfo.getRunId().getId(), ProgramType.SPARK, programId.getProgram(), runId.getId()); }