public void pauseJob(String jobId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } updateJobOutput(jobId, ExecutableState.STOPPED, null, null); }
protected void onExecuteStart(ExecutableContext executableContext) { Map<String, String> info = Maps.newHashMap(); info.put(START_TIME, Long.toString(System.currentTimeMillis())); getManager().updateJobOutput(getId(), ExecutableState.RUNNING, info, null); }
@Override protected void onExecuteStart(ExecutableContext executableContext) { final long startTime = getStartTime(); Map<String, String> info = Maps.newHashMap(); info.put(BUILD_INSTANCE, DistributedLockFactory.processAndHost()); if (startTime > 0) { getManager().updateJobOutput(getId(), ExecutableState.RUNNING, info, null); } else { info.put(START_TIME, Long.toString(System.currentTimeMillis())); getManager().updateJobOutput(getId(), ExecutableState.RUNNING, info, null); } }
@Test public void testValidStateTransfer() throws Exception { SucceedTestExecutable job = new SucceedTestExecutable(); String id = job.getId(); service.addJob(job); service.updateJobOutput(id, ExecutableState.RUNNING, null, null); service.updateJobOutput(id, ExecutableState.ERROR, null, null); service.updateJobOutput(id, ExecutableState.READY, null, null); service.updateJobOutput(id, ExecutableState.RUNNING, null, null); service.updateJobOutput(id, ExecutableState.READY, null, null); service.updateJobOutput(id, ExecutableState.RUNNING, null, null); service.updateJobOutput(id, ExecutableState.SUCCEED, null, null); }
protected void onExecuteError(Throwable exception, ExecutableContext executableContext) { if (!isDiscarded()) { getManager().addJobInfo(getId(), END_TIME, Long.toString(System.currentTimeMillis())); String output = null; if (exception != null) { final StringWriter out = new StringWriter(); exception.printStackTrace(new PrintWriter(out)); output = out.toString(); } getManager().updateJobOutput(getId(), ExecutableState.ERROR, null, output); } }
public void resumeRunningJobForce(String jobId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } if (job instanceof DefaultChainedExecutable) { List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks(); for (AbstractExecutable task : tasks) { if (task.getStatus() == ExecutableState.RUNNING) { updateJobOutput(task.getId(), ExecutableState.READY, null, null); break; } } } updateJobOutput(jobId, ExecutableState.READY, null, null); }
public void rollbackJob(String jobId, String stepId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } if (job instanceof DefaultChainedExecutable) { List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks(); for (AbstractExecutable task : tasks) { if (task.getId().compareTo(stepId) >= 0) { logger.debug("rollback task : " + task); updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), ""); } } } if (job.getStatus() == ExecutableState.SUCCEED) { updateJobOutput(job.getId(), ExecutableState.READY, null, null); } }
public void resumeJob(String jobId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } Map<String, String> info = null; if (job instanceof DefaultChainedExecutable) { List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks(); for (AbstractExecutable task : tasks) { if (task.getStatus() == ExecutableState.ERROR || task.getStatus() == ExecutableState.STOPPED) { updateJobOutput(task.getId(), ExecutableState.READY, null, null); break; } } final long endTime = job.getEndTime(); if (endTime != 0) { long interruptTime = System.currentTimeMillis() - endTime + job.getInterruptTime(); info = Maps.newHashMap(getJobOutput(jobId).getInfo()); info.put(AbstractExecutable.INTERRUPT_TIME, Long.toString(interruptTime)); info.remove(AbstractExecutable.END_TIME); } } updateJobOutput(jobId, ExecutableState.READY, info, null); }
protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) { setEndTime(System.currentTimeMillis()); if (!isDiscarded() && !isRunnable()) { if (result.succeed()) { getManager().updateJobOutput(getId(), ExecutableState.SUCCEED, null, result.output()); } else { getManager().updateJobOutput(getId(), ExecutableState.ERROR, null, result.output()); } } }
@Override protected void onExecuteStart(ExecutableContext executableContext) { final Output output = getOutput(); if (output.getExtra().containsKey(START_TIME)) { final String sparkJobID = output.getExtra().get(ExecutableConstants.SPARK_JOB_ID); if (sparkJobID == null) { getManager().updateJobOutput(getId(), ExecutableState.RUNNING, null, null); return; } try { String status = getAppState(sparkJobID); if (status == null || status.equals("FAILED") || status.equals("KILLED")) { //remove previous mr job info super.onExecuteStart(executableContext); } else { getManager().updateJobOutput(getId(), ExecutableState.RUNNING, null, null); } } catch (IOException e) { logger.warn("error get hadoop status"); super.onExecuteStart(executableContext); } } else { super.onExecuteStart(executableContext); } }
private ExecuteResult onResumed(String appId, ExecutableManager mgr) throws ExecuteException { Map<String, String> info = new HashMap<>(); try { logger.info("spark_job_id:" + appId + " resumed"); info.put(ExecutableConstants.SPARK_JOB_ID, appId); while (!isPaused() && !isDiscarded()) { String status = getAppState(appId); if (status.equals("FAILED") || status.equals("KILLED")) { mgr.updateJobOutput(getId(), ExecutableState.ERROR, null, appId + " has failed"); return new ExecuteResult(ExecuteResult.State.FAILED, appId + " has failed"); } if (status.equals("SUCCEEDED")) { mgr.addJobInfo(getId(), info); return new ExecuteResult(ExecuteResult.State.SUCCEED, appId + " has finished"); } Thread.sleep(5000); } killAppRetry(appId); if (isDiscarded()) { return new ExecuteResult(ExecuteResult.State.DISCARDED, appId + " is discarded"); } else { return new ExecuteResult(ExecuteResult.State.STOPPED, appId + " is stopped"); } } catch (Exception e) { logger.error("error run spark job:", e); return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage()); } }
public void discardJob(String jobId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } if (job.getStatus().isFinalState()) { if (job.getStatus() != ExecutableState.DISCARDED) { logger.warn("The status of job " + jobId + " is " + job.getStatus().toString() + ". It's final state and cannot be transfer to be discarded!!!"); } else { logger.warn("The job " + jobId + " has been discarded."); } return; } if (job instanceof DefaultChainedExecutable) { List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks(); for (AbstractExecutable task : tasks) { if (!task.getStatus().isFinalState()) { updateJobOutput(task.getId(), ExecutableState.DISCARDED, null, null); } } } updateJobOutput(jobId, ExecutableState.DISCARDED, null, null); }
final String mrJobId = output.getExtra().get(ExecutableConstants.MR_JOB_ID); if (mrJobId == null) { getManager().updateJobOutput(getId(), ExecutableState.RUNNING, null, null); return; getManager().updateJobOutput(getId(), ExecutableState.RUNNING, null, null);
@Test(expected = IllegalStateTranferException.class) public void testInvalidStateTransfer() { SucceedTestExecutable job = new SucceedTestExecutable(); service.addJob(job); service.updateJobOutput(job.getId(), ExecutableState.ERROR, null, null); service.updateJobOutput(job.getId(), ExecutableState.STOPPED, null, null); }
public void forceKillJob(String jobId) { try { final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId); List<ExecutablePO> tasks = executableDao.getJob(jobId).getTasks(); for (ExecutablePO task : tasks) { if (executableDao.getJobOutput(task.getId()).getStatus().equals("SUCCEED")) { continue; } else if (executableDao.getJobOutput(task.getId()).getStatus().equals("RUNNING")) { updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), ""); } break; } if (!jobOutput.getStatus().equals(ExecutableState.ERROR.toString())) { jobOutput.setStatus(ExecutableState.ERROR.toString()); executableDao.updateJobOutput(jobOutput); } } catch (PersistentException e) { throw new RuntimeException(e); } }
+ "it might cause endless state, will retry to fetch subtask's state.", task.getId(), task.getName()); getManager().updateJobOutput(task.getId(), ExecutableState.ERROR, null, "killed due to inconsistent state"); hasError = true; mgr.updateJobOutput(getId(), ExecutableState.SUCCEED, null, null); onStatusChange(executableContext, result, ExecutableState.SUCCEED); } else if (hasError) { setEndTime(System.currentTimeMillis()); mgr.updateJobOutput(getId(), ExecutableState.ERROR, null, null); onStatusChange(executableContext, result, ExecutableState.ERROR); } else if (hasDiscarded) { setEndTime(System.currentTimeMillis()); mgr.updateJobOutput(getId(), ExecutableState.DISCARDED, null, null); } else { mgr.updateJobOutput(getId(), ExecutableState.READY, null, null); mgr.updateJobOutput(getId(), ExecutableState.ERROR, null, result.output()); onStatusChange(executableContext, result, ExecutableState.ERROR);
mgr.updateJobOutput(getId(), ExecutableState.ERROR, hadoopCmdOutput.getInfo(), "killed by admin"); return new ExecuteResult(ExecuteResult.State.FAILED, "killed by admin");
@Test public void test() throws Exception { assertNotNull(service); // all existing are broken jobs List<AbstractExecutable> existing = service.getAllExecutables(); for (AbstractExecutable exec : existing) { assertEquals("BrokenExecutable", exec.getClass().getSimpleName()); assertEquals(ExecutableState.DISCARDED, exec.getStatus()); } BaseTestExecutable executable = new SucceedTestExecutable(); executable.setParam("test1", "test1"); executable.setParam("test2", "test2"); executable.setParam("test3", "test3"); service.addJob(executable); List<AbstractExecutable> result = service.getAllExecutables(); assertEquals(existing.size() + 1, result.size()); AbstractExecutable another = service.getJob(executable.getId()); assertJobEqual(executable, another); service.updateJobOutput(executable.getId(), ExecutableState.RUNNING, null, "test output"); assertJobEqual(executable, service.getJob(executable.getId())); }
@Test public void testIllegalState() throws Exception { logger.info("testIllegalState"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); BaseTestExecutable task2 = new RunningTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()).updateJobOutput(task2.getId(), ExecutableState.RUNNING, null, null); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(task2.getId()).getState()); }
@Override protected void onExecuteStart(ExecutableContext executableContext) { final long startTime = getStartTime(); if (startTime > 0) { getManager().updateJobOutput(getId(), ExecutableState.RUNNING, null, null); } else { Map<String, String> info = Maps.newHashMap(); info.put(START_TIME, Long.toString(System.currentTimeMillis())); getManager().updateJobOutput(getId(), ExecutableState.RUNNING, info, null); } }