public Output getOutput(String id) { return getExecutableManager().getOutput(id); }
/** * Get a job step output * * @return * @throws IOException */ @RequestMapping(value = "/{jobId}/steps/{stepId}/output", method = { RequestMethod.GET }, produces = { "application/json" }) @ResponseBody public Map<String, String> getStepOutput(@PathVariable String jobId, @PathVariable String stepId) { Map<String, String> result = new HashMap<String, String>(); result.put("jobId", jobId); result.put("stepId", String.valueOf(stepId)); result.put("cmd_output", jobService.getExecutableManager().getOutput(stepId).getVerboseMsg()); return result; }
public boolean isReady() { final Output output = getManager().getOutput(id); return output.getState() == ExecutableState.READY; }
@Override public final Output getOutput() { return getManager().getOutput(getId()); }
private String findExtraInfo(String key, String dft, boolean backward) { ArrayList<AbstractExecutable> tasks = new ArrayList<AbstractExecutable>(getTasks()); if (backward) { Collections.reverse(tasks); } for (AbstractExecutable child : tasks) { Output output = getManager().getOutput(child.getId()); String value = output.getExtra().get(key); if (value != null) return value; } return dft; } }
@Override public ExecutableState getStatus() { ExecutableManager manager = getManager(); return manager.getOutput(this.getId()).getState(); }
@Override public boolean isReady() { if (!super.isReady()) { return false; } for (Executable task : subTasksForCheck) { final Output output = getManager().getOutput(task.getId()); if (output.getState() != ExecutableState.SUCCEED) { return false; } } return true; }
@Override public void onUnlock(String path, String nodeData) { String[] paths = StringUtil.split(path, "/"); String jobId = paths[paths.length - 1]; // Sync execute cache in case broadcast not available try { executableManager.syncDigestsOfJob(jobId); } catch (PersistentException e) { logger.error("Failed to sync cache of job: " + jobId + ", at server: " + serverName); } final Output output = executableManager.getOutput(jobId); if (output.getState() == ExecutableState.RUNNING) { AbstractExecutable executable = executableManager.getJob(jobId); if (executable instanceof DefaultChainedExecutable && !nodeData.equalsIgnoreCase(serverName)) { try { logger.warn(nodeData + " has released the lock for: " + jobId + " but the job still running. so " + serverName + " resume the job"); if (!jobLock.isLocked(getLockPath(jobId))) { executableManager.resumeRunningJobForce(executable.getId()); fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS); } } catch (Exception e) { logger.error("resume the job but fail in server: " + serverName, e); } } } }
private void resumeAllRunningJobs() { for (final String id : executableManager.getAllJobIds()) { final Output output = executableManager.getOutput(id); AbstractExecutable executable = executableManager.getJob(id); if (output.getState() == ExecutableState.RUNNING && executable instanceof DefaultChainedExecutable) { try { if (!jobLock.isLocked(getLockPath(executable.getId()))) { executableManager.resumeRunningJobForce(executable.getId()); fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS); } } catch (Exception e) { logger.error("resume the job " + id + " fail in server: " + serverName, e); } } } }
public void calculateCardinalityIfNotPresent(String[] tables, String submitter, String prj) throws Exception { // calculate cardinality for Hive source ProjectInstance projectInstance = getProjectManager().getProject(prj); if (projectInstance == null || projectInstance.getSourceType() != ISourceAware.ID_HIVE){ return; } TableMetadataManager metaMgr = getTableManager(); ExecutableManager exeMgt = ExecutableManager.getInstance(getConfig()); for (String table : tables) { TableExtDesc tableExtDesc = metaMgr.getTableExt(table, prj); String jobID = tableExtDesc.getJodID(); if (null == jobID || ExecutableState.RUNNING != exeMgt.getOutput(jobID).getState()) { calculateCardinality(table, submitter, prj); } } }
@Test public void testSingleTaskJob() throws Exception { logger.info("testSingleTaskJob"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); job.addTask(task1); execMgr.addJob(job); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); }
@Test public void testSucceed() throws Exception { logger.info("testSucceed"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); BaseTestExecutable task2 = new SucceedTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task2.getId()).getState()); }
@Test public void testSucceedAndFailed() throws Exception { logger.info("testSucceedAndFailed"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); BaseTestExecutable task2 = new FailedTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(task2.getId()).getState()); }
@Test public void testSucceedAndError() throws Exception { logger.info("testSucceedAndError"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new ErrorTestExecutable(); BaseTestExecutable task2 = new SucceedTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.READY, execMgr.getOutput(task2.getId()).getState()); }
@Test public void testSchedulerRestart() throws Exception { logger.info("testSchedulerRestart"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new FiveSecondSucceedTestExecutable(); job.addTask(task1); execMgr.addJob(job); //sleep 3s to make sure SucceedTestExecutable is running Thread.sleep(3000); //scheduler failed due to some reason scheduler.shutdown(); //restart startScheduler(); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); }
@Test public void testMetaStoreRecover() throws Exception { logger.info("testMetaStoreRecover"); NoErrorStatusExecutable job = new NoErrorStatusExecutable(); ErrorTestExecutable task = new ErrorTestExecutable(); job.addTask(task); execMgr.addJob(job); Thread.sleep(2500); runningJobToError(job.getId()); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); }
@Test public void testDiscard() throws Exception { logger.info("testDiscard"); DefaultChainedExecutable job = new DefaultChainedExecutable(); SelfStopExecutable task1 = new SelfStopExecutable(); job.addTask(task1); execMgr.addJob(job); Thread.sleep(1100); // give time to launch job/task1 waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500); execMgr.discardJob(job.getId()); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.DISCARDED, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.DISCARDED, execMgr.getOutput(task1.getId()).getState()); task1.waitForDoWork(); }
@Test public void testIllegalState() throws Exception { logger.info("testIllegalState"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); BaseTestExecutable task2 = new RunningTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()).updateJobOutput(task2.getId(), ExecutableState.RUNNING, null, null); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(task2.getId()).getState()); }
protected JobInstance getCheckpointJobInstance(AbstractExecutable job) { Message msg = MsgPicker.getMsg(); if (job == null) { return null; } if (!(job instanceof CheckpointExecutable)) { throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_JOB_TYPE(), job.getId())); } CheckpointExecutable checkpointExecutable = (CheckpointExecutable) job; Output output = checkpointExecutable.getOutput(); final JobInstance result = new JobInstance(); result.setName(job.getName()); result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams())); result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams())); result.setLastModified(job.getLastModified()); result.setSubmitter(job.getSubmitter()); result.setUuid(job.getId()); result.setType(CubeBuildTypeEnum.CHECKPOINT); result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus())); result.setBuildInstance(AbstractExecutable.getBuildInstance(output)); result.setDuration(job.getDuration() / 1000); for (int i = 0; i < checkpointExecutable.getTasks().size(); ++i) { AbstractExecutable task = checkpointExecutable.getTasks().get(i); result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId()))); } return result; }
protected JobInstance getLookupSnapshotBuildJobInstance(LookupSnapshotBuildJob job) { if (job == null) { return null; } Output output = job.getOutput(); final JobInstance result = new JobInstance(); result.setName(job.getName()); result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams())); result.setRelatedSegment(CubingExecutableUtil.getSegmentId(job.getParams())); result.setLastModified(job.getLastModified()); result.setSubmitter(job.getSubmitter()); result.setUuid(job.getId()); result.setType(CubeBuildTypeEnum.BUILD); result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus())); result.setBuildInstance(AbstractExecutable.getBuildInstance(output)); result.setDuration(job.getDuration() / 1000); for (int i = 0; i < job.getTasks().size(); ++i) { AbstractExecutable task = job.getTasks().get(i); result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId()))); } return result; }