public void resumeJob(String jobId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } Map<String, String> info = null; if (job instanceof DefaultChainedExecutable) { List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks(); for (AbstractExecutable task : tasks) { if (task.getStatus() == ExecutableState.ERROR || task.getStatus() == ExecutableState.STOPPED) { updateJobOutput(task.getId(), ExecutableState.READY, null, null); break; } } final long endTime = job.getEndTime(); if (endTime != 0) { long interruptTime = System.currentTimeMillis() - endTime + job.getInterruptTime(); info = Maps.newHashMap(getJobOutput(jobId).getInfo()); info.put(AbstractExecutable.INTERRUPT_TIME, Long.toString(interruptTime)); info.remove(AbstractExecutable.END_TIME); } } updateJobOutput(jobId, ExecutableState.READY, info, null); }
public final long getDuration() { return getDuration(getStartTime(), getEndTime(), getInterruptTime()); }
@Override public final ExecuteResult execute(ExecutableContext executableContext) throws ExecuteException { logger.info("Executing AbstractExecutable ({})", this.getName()); onExecuteStart(executableContext); Throwable catchedException; Throwable realException; result = null; try { result = doWork(executableContext); } catch (Throwable e) { logger.error("error running Executable: {}", this.toString()); catchedException = e; } while (needRetry(this.retry, realException)); //exception in ExecuteResult should handle by user itself. onExecuteError(realException, executableContext); throw new ExecuteException(realException); onExecuteFinishedWithRetry(result, executableContext); } catch (ExecuteException e) { throw e;
protected long getExtraInfoAsLong(String key, long defaultValue) { return getExtraInfoAsLong(getOutput(), key, defaultValue); }
public static JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output stepOutput) { JobInstance.JobStep result = new JobInstance.JobStep(); result.setId(task.getId()); result.setName(task.getName()); result.setSequenceID(i); logger.warn("Cannot found output for task: id={}", task.getId()); return result; result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput)); result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput)); if (task instanceof ShellExecutable) { result.setExecCmd(((ShellExecutable) task).getCmd()); result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams()); result.setExecWaitTime( AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
protected JobInstance getCheckpointJobInstance(AbstractExecutable job) { Message msg = MsgPicker.getMsg(); if (job == null) { return null; } if (!(job instanceof CheckpointExecutable)) { throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_JOB_TYPE(), job.getId())); } CheckpointExecutable checkpointExecutable = (CheckpointExecutable) job; Output output = checkpointExecutable.getOutput(); final JobInstance result = new JobInstance(); result.setName(job.getName()); result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams())); result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams())); result.setLastModified(job.getLastModified()); result.setSubmitter(job.getSubmitter()); result.setUuid(job.getId()); result.setType(CubeBuildTypeEnum.CHECKPOINT); result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus())); result.setBuildInstance(AbstractExecutable.getBuildInstance(output)); result.setDuration(job.getDuration() / 1000); for (int i = 0; i < checkpointExecutable.getTasks().size(); ++i) { AbstractExecutable task = checkpointExecutable.getTasks().get(i); result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId()))); } return result; }
private static ExecutablePO parse(AbstractExecutable executable) { ExecutablePO result = new ExecutablePO(); result.setName(executable.getName()); result.setUuid(executable.getId()); result.setType(executable.getClass().getName()); result.setParams(executable.getParams()); if (executable instanceof ChainedExecutable) { List<ExecutablePO> tasks = Lists.newArrayList(); for (AbstractExecutable task : ((ChainedExecutable) executable).getTasks()) { tasks.add(parse(task)); } result.setTasks(tasks); } if (executable instanceof CheckpointExecutable) { List<ExecutablePO> tasksForCheck = Lists.newArrayList(); for (AbstractExecutable taskForCheck : ((CheckpointExecutable) executable).getSubTasksForCheck()) { tasksForCheck.add(parse(taskForCheck)); } result.setTasksForCheck(tasksForCheck); } return result; }
public void rollbackJob(String jobId, String stepId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } if (job instanceof DefaultChainedExecutable) { List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks(); for (AbstractExecutable task : tasks) { if (task.getId().compareTo(stepId) >= 0) { logger.debug("rollback task : " + task); updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), ""); } } } if (job.getStatus() == ExecutableState.SUCCEED) { updateJobOutput(job.getId(), ExecutableState.READY, null, null); } }
@Override public final ExecuteResult execute(ExecutableContext executableContext) throws ExecuteException { //print a eye-catching title in log LogTitlePrinter.printTitle(this.getName()); Preconditions.checkArgument(executableContext instanceof DefaultContext); ExecuteResult result; try { onExecuteStart(executableContext); result = doWork(executableContext); } catch (Throwable e) { logger.error("error running Executable", e); onExecuteError(e, executableContext); throw new ExecuteException(e); } onExecuteFinished(result, executableContext); return result; }
throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_JOB_TYPE(), job.getId())); Output output = cubeJob.getOutput(); final JobInstance result = new JobInstance(); result.setName(job.getName()); if (cube != null) { result.setRelatedCube(cube.getName()); result.setUuid(cubeJob.getId()); result.setType(CubeBuildTypeEnum.BUILD); result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus())); result.setMrWaiting(cubeJob.getMapReduceWaitTime() / 1000); result.setBuildInstance(AbstractExecutable.getBuildInstance(output)); result.setDuration(cubeJob.getDuration() / 1000); for (int i = 0; i < cubeJob.getTasks().size(); ++i) { AbstractExecutable task = cubeJob.getTasks().get(i); result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
public final String getSubmitter() { return getParam(SUBMITTER); }
@Override public boolean isRunnable() { return this.getStatus() == ExecutableState.READY; }
private void releaseJobLock(AbstractExecutable executable) { if (executable instanceof DefaultChainedExecutable) { ExecutableState state = executable.getStatus(); if (state != ExecutableState.READY && state != ExecutableState.RUNNING) { if (jobWithLocks.contains(executable.getId())) { logger.info( executable.toString() + " will release the lock for the job: " + executable.getId()); jobLock.unlock(getLockPath(executable.getId())); jobWithLocks.remove(executable.getId()); } } } } }
public final long getInterruptTime() { return getExtraInfoAsLong(INTERRUPT_TIME, 0L); }
protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) { setEndTime(System.currentTimeMillis()); if (!isDiscarded() && !isRunnable()) { if (result.succeed()) { getManager().updateJobOutput(getId(), ExecutableState.SUCCEED, null, result.output()); } else { getManager().updateJobOutput(getId(), ExecutableState.ERROR, null, result.output()); } } }
private void doSendMail(KylinConfig kylinConfig, List<String> users, Pair<String, String> email) { if (email == null) { logger.warn("no need to send email, content is null"); return; } logger.info("prepare to send email to:{}", users); logger.info("job name:{}", getName()); logger.info("submitter:{}", getSubmitter()); logger.info("notify list:{}", users); new MailService(kylinConfig).sendMail(users, email.getFirst(), email.getSecond()); }
public final long getDuration() { return getDuration(getStartTime(), getEndTime()); }
protected final void notifyUserStatusChange(ExecutableState state) { try { List<String> users = Lists.newArrayList(); users.addAll(getNotifyList()); final String adminDls = KylinConfig.getInstanceFromEnv().getAdminDls(); if (null != adminDls) { for (String adminDl : adminDls.split(",")) { users.add(adminDl); } } if (users.isEmpty()) { return; } final Pair<String, String> email = formatNotifications(state); if (email == null) { return; } logger.info("prepare to send email to:" + users); logger.info("job name:" + getName()); logger.info("submitter:" + getSubmitter()); logger.info("notify list:" + users); new MailService().sendMail(users, email.getLeft(), email.getRight()); } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); } }
@Override public final Output getOutput() { return getManager().getOutput(getId()); }