private void releaseJobLock(AbstractExecutable executable) { if (executable instanceof DefaultChainedExecutable) { ExecutableState state = executable.getStatus(); if (state != ExecutableState.READY && state != ExecutableState.RUNNING) { if (jobWithLocks.contains(executable.getId())) { logger.info( executable.toString() + " will release the lock for the job: " + executable.getId()); jobLock.unlock(getLockPath(executable.getId())); jobWithLocks.remove(executable.getId()); } } } } }
@Override public String toString() { return Objects.toStringHelper(this).add("id", getId()).add("name", getName()).add("state", getStatus()) .toString(); } }
protected void onExecuteStart(ExecutableContext executableContext) { Map<String, String> info = Maps.newHashMap(); info.put(START_TIME, Long.toString(System.currentTimeMillis())); getManager().updateJobOutput(getId(), ExecutableState.RUNNING, info, null); }
public void addJob(AbstractExecutable executable) { try { executable.initConfig(config); if (executableDao.getJob(executable.getId()) != null) { throw new IllegalArgumentException("job id:" + executable.getId() + " already exists"); } addJobOutput(executable); executableDao.addJob(parse(executable)); } catch (PersistentException e) { logger.error("fail to submit job:" + executable.getId(), e); throw new RuntimeException(e); } }
protected void onExecuteError(Throwable exception, ExecutableContext executableContext) { if (!isDiscarded()) { getManager().addJobInfo(getId(), END_TIME, Long.toString(System.currentTimeMillis())); String output = null; if (exception != null) { final StringWriter out = new StringWriter(); exception.printStackTrace(new PrintWriter(out)); output = out.toString(); } getManager().updateJobOutput(getId(), ExecutableState.ERROR, null, output); } }
private void setRelatedIdList(CheckpointExecutable checkpointExecutable, List<String> segmentIdList, List<String> jobIdList) { for (AbstractExecutable taskForCheck : checkpointExecutable.getSubTasksForCheck()) { jobIdList.add(taskForCheck.getId()); if (taskForCheck instanceof CubingJob) { segmentIdList.addAll(Lists .newArrayList(StringUtils.split(CubingExecutableUtil.getSegmentId(taskForCheck.getParams())))); } else if (taskForCheck instanceof CheckpointExecutable) { setRelatedIdList((CheckpointExecutable) taskForCheck, segmentIdList, jobIdList); } } }
public void rollbackJob(String jobId, String stepId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } if (job instanceof DefaultChainedExecutable) { List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks(); for (AbstractExecutable task : tasks) { if (task.getId().compareTo(stepId) >= 0) { logger.debug("rollback task : " + task); updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), ""); } } } if (job.getStatus() == ExecutableState.SUCCEED) { updateJobOutput(job.getId(), ExecutableState.READY, null, null); } }
@Override public void run() { try (SetThreadName ignored = new SetThreadName("Scheduler %s Job %s", System.identityHashCode(DistributedScheduler.this), executable.getId())) { if (jobLock.lock(getLockPath(executable.getId()))) { logger.info(executable.toString() + " scheduled in server: " + serverName); context.addRunningJob(executable); jobWithLocks.add(executable.getId()); executable.execute(context); } } catch (ExecuteException e) { logger.error("ExecuteException job:" + executable.getId() + " in server: " + serverName, e); } catch (Exception e) { logger.error("unknown error execute job:" + executable.getId() + " in server: " + serverName, e); } finally { context.removeRunningJob(executable); releaseJobLock(executable); // trigger the next step asap fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS); } }
@Override public final Output getOutput() { return getManager().getOutput(getId()); }
@Override public void run() { try (SetThreadName ignored = new SetThreadName("Scheduler %s Job %s", System.identityHashCode(DefaultScheduler.this), executable.getId())) { executable.execute(context); } catch (ExecuteException e) { logger.error("ExecuteException job:" + executable.getId(), e); } catch (Exception e) { if (AbstractExecutable.isMetaDataPersistException(e, 5)) { // Job fail due to PersistException ExecutableManager.getInstance(jobEngineConfig.getConfig()) .forceKillJobWithRetry(executable.getId()); } logger.error("unknown error execute job:" + executable.getId(), e); } finally { context.removeRunningJob(executable); } // trigger the next step asap fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS); } }
public final void addExtraInfo(String key, String value) { getManager().addJobInfo(getId(), key, value); }
private String findExtraInfo(String key, String dft, boolean backward) { ArrayList<AbstractExecutable> tasks = new ArrayList<AbstractExecutable>(getTasks()); if (backward) { Collections.reverse(tasks); } for (AbstractExecutable child : tasks) { Output output = getManager().getOutput(child.getId()); String value = output.getExtra().get(key); if (value != null) return value; } return dft; } }
@Override public ExecutableState getStatus() { ExecutableManager manager = getManager(); return manager.getOutput(this.getId()).getState(); }
private void resumeAllRunningJobs() { for (final String id : executableManager.getAllJobIds()) { final Output output = executableManager.getOutput(id); AbstractExecutable executable = executableManager.getJob(id); if (output.getState() == ExecutableState.RUNNING && executable instanceof DefaultChainedExecutable) { try { if (!jobLock.isLocked(getLockPath(executable.getId()))) { executableManager.resumeRunningJobForce(executable.getId()); fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS); } } catch (Exception e) { logger.error("resume the job " + id + " fail in server: " + serverName, e); } } } }
protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) { setEndTime(System.currentTimeMillis()); if (!isDiscarded() && !isRunnable()) { if (result.succeed()) { getManager().updateJobOutput(getId(), ExecutableState.SUCCEED, null, result.output()); } else { getManager().updateJobOutput(getId(), ExecutableState.ERROR, null, result.output()); } } }
private void addJobOutput(AbstractExecutable executable) throws PersistentException { ExecutableOutputPO executableOutputPO = new ExecutableOutputPO(); executableOutputPO.setUuid(executable.getId()); executableDao.addJobOutput(executableOutputPO); if (executable instanceof DefaultChainedExecutable) { for (AbstractExecutable subTask : ((DefaultChainedExecutable) executable).getTasks()) { addJobOutput(subTask); } } }
private static ExecutablePO parse(AbstractExecutable executable) { ExecutablePO result = new ExecutablePO(); result.setName(executable.getName()); result.setUuid(executable.getId()); result.setType(executable.getClass().getName()); result.setParams(executable.getParams()); if (executable instanceof ChainedExecutable) { List<ExecutablePO> tasks = Lists.newArrayList(); for (AbstractExecutable task : ((ChainedExecutable) executable).getTasks()) { tasks.add(parse(task)); } result.setTasks(tasks); } if (executable instanceof CheckpointExecutable) { List<ExecutablePO> tasksForCheck = Lists.newArrayList(); for (AbstractExecutable taskForCheck : ((CheckpointExecutable) executable).getSubTasksForCheck()) { tasksForCheck.add(parse(taskForCheck)); } result.setTasksForCheck(tasksForCheck); } return result; }
public void resumeRunningJobForce(String jobId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } if (job instanceof DefaultChainedExecutable) { List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks(); for (AbstractExecutable task : tasks) { if (task.getStatus() == ExecutableState.RUNNING) { updateJobOutput(task.getId(), ExecutableState.READY, null, null); break; } } } updateJobOutput(jobId, ExecutableState.READY, null, null); }
public void cancelJob(JobInstance job) throws IOException { aclEvaluate.checkProjectOperationPermission(job); if (null == job.getRelatedCube() || null == getCubeManager().getCube(job.getRelatedCube()) || null == job.getRelatedSegment()) { getExecutableManager().discardJob(job.getId()); } logger.info("Cancel job [" + job.getId() + "] trigger by " + SecurityContextHolder.getContext().getAuthentication().getName()); if (job.getStatus() == JobStatusEnum.FINISHED) { throw new IllegalStateException( "The job " + job.getId() + " has already been finished and cannot be discarded."); } if (job.getStatus() != JobStatusEnum.DISCARDED) { AbstractExecutable executable = getExecutableManager().getJob(job.getId()); if (executable instanceof CubingJob) { cancelCubingJobInner((CubingJob) executable); } else if (executable instanceof CheckpointExecutable) { cancelCheckpointJobInner((CheckpointExecutable) executable); } else { getExecutableManager().discardJob(executable.getId()); } } }
protected JobInstance getLookupSnapshotBuildJobInstance(LookupSnapshotBuildJob job) { if (job == null) { return null; } Output output = job.getOutput(); final JobInstance result = new JobInstance(); result.setName(job.getName()); result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams())); result.setRelatedSegment(CubingExecutableUtil.getSegmentId(job.getParams())); result.setLastModified(job.getLastModified()); result.setSubmitter(job.getSubmitter()); result.setUuid(job.getId()); result.setType(CubeBuildTypeEnum.BUILD); result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus())); result.setBuildInstance(AbstractExecutable.getBuildInstance(output)); result.setDuration(job.getDuration() / 1000); for (int i = 0; i < job.getTasks().size(); ++i) { AbstractExecutable task = job.getTasks().get(i); result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId()))); } return result; }