@Override String getKey(ExecutionContext context, ExecutionScript script) { return MessageFormat.format( "{0}.{1}", context.getFlowId(), WILDCARD); } },
@Override String getKey(ExecutionContext context, ExecutionScript script) { return MessageFormat.format( "{0}.{1}.{2}", context.getFlowId(), context.getPhase().getSymbol(), WILDCARD); } },
@Override String getKey(ExecutionContext context, ExecutionScript script) { if (script == null) { return null; } return MessageFormat.format( "{0}.{1}.{2}", context.getFlowId(), context.getPhase().getSymbol(), script.getId()); } },
private List<String> buildExecutionCommand( ExecutionContext context, HadoopScript script) throws IOException, InterruptedException { assert context != null; assert script != null; List<String> command = new ArrayList<>(); command.add(getCommand(context, PATH_EXECUTE, script)); command.add(script.getClassName()); command.add(context.getBatchId()); command.add(context.getFlowId()); command.add(context.getExecutionId()); command.add(context.getArgumentsAsString()); Map<String, String> props = buildHadoopProperties(context, script); for (Map.Entry<String, String> entry : props.entrySet()) { command.add("-D"); command.add(MessageFormat.format("{0}={1}", entry.getKey(), entry.getValue())); } return command; }
/** * Computes the tracking ID for script-less job. * @param context target context * @return the operation ID * @since 0.5.0 */ public static String computeTrackingId(ExecutionContext context) { return MessageFormat.format( "YAESS/{0}/{1}/{2}/{3}", context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId()); }
/** * Creates a new instance. * @param context current context * @param stepUnit report unit for each progress size (0.0~1.0) * @throws IllegalArgumentException if some parameters were {@code null} */ public LoggingExecutionMonitor(ExecutionContext context, double stepUnit) { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ } this.label = MessageFormat.format("{0}|{1}|{3}@{2}", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase()); if (stepUnit <= 0) { this.stepUnit = Double.MAX_VALUE; } else { this.stepUnit = Math.max(stepUnit, 0.01) - 0.0000000000001; } }
private ExecutionScriptHandler<T> resolve( ExecutionContext context, ExecutionScript script) throws IOException { assert context != null; Properties batchConf = getBatchConf(context, script); String key = findKey(context, script, batchConf); if (key != null) { ExecutionScriptHandler<T> target = delegations.get(key); if (target != null) { return target; } throw new IOException(MessageFormat.format( "Invalid dispatch target for multidispatch plugin: " + "{4} (batchId={0}, flowId={1}, phase={2}, stageId={3})", context.getBatchId(), context.getFlowId(), context.getPhase(), script == null ? LABEL_UNDEFINED : script.getId(), key)); } ExecutionScriptHandler<T> defaultTarget = delegations.get(PREFIX_DEFAULT); assert defaultTarget != null; return defaultTarget; }
"Jobnet execution was deadlocked: {4} (batch={0}, flow={1}, phase={2}, execution={3})", context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId(), "Jobnet was failed by preceding errors (batch={0}, flow={1}, phase={2}, execution={3})", context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId()));
/** * Computes the tracking ID. * @param context target context * @param script target script * @return the operation ID * @since 0.5.0 */ public static String computeTrackingId(ExecutionContext context, ExecutionScript script) { return MessageFormat.format( "YAESS/{0}/{1}/{2}/{3}/{4}", context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId(), script.getId()); }
/** * Executes a target phase. * @param context the current context * @throws InterruptedException if interrupted during this execution * @throws IOException if failed to execute target phase * @throws IllegalArgumentException if some parameters were {@code null} * @since 0.2.5 */ public void executePhase(ExecutionContext context) throws InterruptedException, IOException { if (context == null) { throw new IllegalArgumentException("context must not be null"); //$NON-NLS-1$ } FlowScript flow = script.findFlow(context.getFlowId()); if (flow == null) { throw new IllegalArgumentException(MessageFormat.format( "Flow is undefined: batchId={0}, flowId={1}, executionId={2}", context.getBatchId(), context.getFlowId(), context.getExecutionId())); } Set<ExecutionScript> executions = flow.getScripts().get(context.getPhase()); try (ExecutionLock lock = acquireExecutionLock(context.getBatchId())) { lock.beginFlow(context.getFlowId(), context.getExecutionId()); executePhase(context, executions, flow.getEnabledScriptKinds()); lock.endFlow(context.getFlowId(), context.getExecutionId()); } }
new Object[] { context.getBatchId(), context.getFlowId(), context.getPhase(), script == null ? LABEL_UNDEFINED : script.getId(), LOG.debug("Dispatch target does not found (batchId={}, flowId={}, phase={}, stageId={})", new Object[] { context.getBatchId(), context.getFlowId(), context.getPhase().getSymbol(), script == null ? LABEL_UNDEFINED : script.getId(),
/** * Performs as {@link #setUp(ExecutionMonitor, ExecutionContext)} that does nothing. * @param context current context * @since 0.4.0 */ protected final void voidSetUp(ExecutionContext context) { YSLOG.info("I51001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getHandlerId()); }
private void command( ExecutionMonitor monitor, ExecutionContext context, ExecutionScript script, List<String> command) throws InterruptedException, IOException { assert monitor != null; assert context != null; assert command != null; assert command.isEmpty() == false; Map<String, String> env = getEnvironmentVariables(context, script); LOG.debug("env: {}", env); LOG.debug("command: {}", command); Map<String, Blob> extensions = BlobUtil.getExtensions(context, script); LOG.debug("extensions: {}", extensions); monitor.checkCancelled(); ProcessExecutor executor = getCommandExecutor(); int exit = executor.execute(context, command, env, extensions, monitor.getOutput()); if (exit == 0) { return; } throw new ExitCodeException(MessageFormat.format( "Unexpected exit code from command job: " + "code={4} (batch={0}, flow={1}, phase={2}, exection={3})", context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId(), String.valueOf(exit)), exit); }
/** * Performs as {@link #cleanUp(ExecutionMonitor, ExecutionContext)} that does nothing. * @param context current context * @since 0.4.0 */ protected final void voidCleanUp(ExecutionContext context) { YSLOG.info("I51002", context.getBatchId(), context.getFlowId(), context.getExecutionId(), context.getPhase(), getHandlerId()); } }
@Override public void cleanUp(ExecutionMonitor monitor, ExecutionContext context) throws InterruptedException, IOException { ExecutionScriptHandler<T> target; if (forceCleanUp != null) { target = delegations.get(forceCleanUp); } else { target = resolve(context, null); } assert target != null; YSLOG.info("I01003", target.getHandlerId(), context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId()); target.cleanUp(monitor, context); }
@Override public void setUp(ExecutionMonitor monitor, ExecutionContext context) throws InterruptedException, IOException { ExecutionScriptHandler<T> target; if (forceSetUp != null) { target = delegations.get(forceSetUp); } else { target = resolve(context, null); } assert target != null; YSLOG.info("I01001", target.getHandlerId(), context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId()); target.setUp(monitor, context); }
"Failed to build command: {6} (batch={0}, flow={1}, phase={3}, stage={4}, execution={2})", context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId(), + "code={5} (batch={0}, flow={1}, phase={2}, stage={4}, exection={3})", context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId(),
@Override public void cleanUp( ExecutionMonitor monitor, ExecutionContext context) throws InterruptedException, IOException { monitor.open(1); try { if (cleanup) { YSLOG.info("I51001", context.getBatchId(), context.getFlowId(), context.getExecutionId(), getHandlerId()); HadoopScript script = new HadoopScript( context.getPhase().getSymbol(), Collections.emptySet(), CLEANUP_STAGE_CLASS, Collections.emptyMap(), Collections.emptyMap()); execute0(monitor, context, script); } else { YSLOG.info("I51002", context.getBatchId(), context.getFlowId(), context.getExecutionId(), getHandlerId()); } } finally { monitor.close(); } }
private JobScript convert(ExecutionContext context, HadoopScript script) throws InterruptedException, IOException { assert context != null; assert script != null; JobScript result = new JobScript(); result.setBatchId(context.getBatchId()); result.setFlowId(context.getFlowId()); result.setExecutionId(context.getExecutionId()); result.setPhase(context.getPhase()); result.setArguments(new HashMap<>(context.getArguments())); result.setStageId(script.getId()); result.setMainClassName(script.getClassName()); Map<String, String> props = new HashMap<>(); props.putAll(getProperties(context, script)); props.putAll(script.getHadoopProperties()); props.put(HadoopScriptUtil.PROP_TRACKING_ID, Job.computeTrackingId(context, script)); result.setProperties(props); Map<String, String> env = new HashMap<>(); // NOTE: Handler has only dummy environment variables // env.putAll(getEnvironmentVariables(context, script)); env.putAll(context.getEnvironmentVariables()); env.putAll(script.getEnvironmentVariables()); result.setEnvironmentVariables(env); return result; }
@Override public void execute( ExecutionMonitor monitor, ExecutionContext context, T script) throws InterruptedException, IOException { ExecutionScriptHandler<T> target = resolve(context, script); assert target != null; YSLOG.info("I01002", target.getHandlerId(), context.getBatchId(), context.getFlowId(), context.getPhase(), context.getExecutionId(), script.getId()); target.execute(monitor, context, script); }