protected final boolean isDiscarded() { final ExecutableState status = getOutput().getState(); return status == ExecutableState.DISCARDED; }
protected final boolean isPaused() { final ExecutableState status = getOutput().getState(); return status == ExecutableState.STOPPED; }
@Override public boolean apply(CubingJob executable) { try { Output output = allOutputs.get(executable.getId()); if (output == null) { return false; } ExecutableState state = output.getState(); boolean ret = statusList.contains(state); return ret; } catch (Exception e) { throw e; } } }, new Predicate<CubingJob>() {
@Override public boolean apply(CheckpointExecutable executable) { try { Output output = allOutputs.get(executable.getId()); if (output == null) { return false; } ExecutableState state = output.getState(); boolean ret = statusList.contains(state); return ret; } catch (Exception e) { throw e; } } }, new Predicate<CheckpointExecutable>() {
public boolean isReady() { final Output output = getManager().getOutput(id); return output.getState() == ExecutableState.READY; }
@Override public ExecutableState getStatus() { ExecutableManager manager = getManager(); return manager.getOutput(this.getId()).getState(); }
@Override public boolean isReady() { if (!super.isReady()) { return false; } for (Executable task : subTasksForCheck) { final Output output = getManager().getOutput(task.getId()); if (output.getState() != ExecutableState.SUCCEED) { return false; } } return true; }
@Override public void onUnlock(String path, String nodeData) { String[] paths = StringUtil.split(path, "/"); String jobId = paths[paths.length - 1]; // Sync execute cache in case broadcast not available try { executableManager.syncDigestsOfJob(jobId); } catch (PersistentException e) { logger.error("Failed to sync cache of job: " + jobId + ", at server: " + serverName); } final Output output = executableManager.getOutput(jobId); if (output.getState() == ExecutableState.RUNNING) { AbstractExecutable executable = executableManager.getJob(jobId); if (executable instanceof DefaultChainedExecutable && !nodeData.equalsIgnoreCase(serverName)) { try { logger.warn(nodeData + " has released the lock for: " + jobId + " but the job still running. so " + serverName + " resume the job"); if (!jobLock.isLocked(getLockPath(jobId))) { executableManager.resumeRunningJobForce(executable.getId()); fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS); } } catch (Exception e) { logger.error("resume the job but fail in server: " + serverName, e); } } } }
private void resumeAllRunningJobs() { for (final String id : executableManager.getAllJobIds()) { final Output output = executableManager.getOutput(id); AbstractExecutable executable = executableManager.getJob(id); if (output.getState() == ExecutableState.RUNNING && executable instanceof DefaultChainedExecutable) { try { if (!jobLock.isLocked(getLockPath(executable.getId()))) { executableManager.resumeRunningJobForce(executable.getId()); fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS); } } catch (Exception e) { logger.error("resume the job " + id + " fail in server: " + serverName, e); } } } }
private JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output stepOutput) { Preconditions.checkNotNull(stepOutput); JobInstance.JobStep result = new JobInstance.JobStep(); result.setId(task.getId()); result.setName(task.getName()); result.setSequenceID(i); result.setStatus(parseToJobStepStatus(stepOutput.getState())); for (Map.Entry<String, String> entry : stepOutput.getExtra().entrySet()) { if (entry.getKey() != null && entry.getValue() != null) { result.putInfo(entry.getKey(), entry.getValue()); } } result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput)); result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput)); if (task instanceof ShellExecutable) { result.setExecCmd(((ShellExecutable) task).getCmd()); } if (task instanceof MapReduceExecutable) { result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams()); result.setExecWaitTime(AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME, 0L) / 1000); } if (task instanceof HadoopShellExecutable) { result.setExecCmd(((HadoopShellExecutable) task).getJobParams()); } return result; }
@Test public void testSingleTaskJob() throws Exception { logger.info("testSingleTaskJob"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); job.addTask(task1); execMgr.addJob(job); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); }
public void calculateCardinalityIfNotPresent(String[] tables, String submitter, String prj) throws Exception { // calculate cardinality for Hive source ProjectInstance projectInstance = getProjectManager().getProject(prj); if (projectInstance == null || projectInstance.getSourceType() != ISourceAware.ID_HIVE){ return; } TableMetadataManager metaMgr = getTableManager(); ExecutableManager exeMgt = ExecutableManager.getInstance(getConfig()); for (String table : tables) { TableExtDesc tableExtDesc = metaMgr.getTableExt(table, prj); String jobID = tableExtDesc.getJodID(); if (null == jobID || ExecutableState.RUNNING != exeMgt.getOutput(jobID).getState()) { calculateCardinality(table, submitter, prj); } } }
@Test public void testSucceedAndError() throws Exception { logger.info("testSucceedAndError"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new ErrorTestExecutable(); BaseTestExecutable task2 = new SucceedTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.READY, execMgr.getOutput(task2.getId()).getState()); }
@Test public void testSucceed() throws Exception { logger.info("testSucceed"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); BaseTestExecutable task2 = new SucceedTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task2.getId()).getState()); }
@Test public void testSucceedAndFailed() throws Exception { logger.info("testSucceedAndFailed"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); BaseTestExecutable task2 = new FailedTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(task2.getId()).getState()); }
@Test public void testSchedulerRestart() throws Exception { logger.info("testSchedulerRestart"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new FiveSecondSucceedTestExecutable(); job.addTask(task1); execMgr.addJob(job); //sleep 3s to make sure SucceedTestExecutable is running Thread.sleep(3000); //scheduler failed due to some reason scheduler.shutdown(); //restart startScheduler(); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); }
@Test public void testMetaStoreRecover() throws Exception { logger.info("testMetaStoreRecover"); NoErrorStatusExecutable job = new NoErrorStatusExecutable(); ErrorTestExecutable task = new ErrorTestExecutable(); job.addTask(task); execMgr.addJob(job); Thread.sleep(2500); runningJobToError(job.getId()); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); }
result.setUuid(job.getId()); result.setType(CubeBuildTypeEnum.CHECKPOINT); result.setStatus(parseToJobStatus(output.getState())); result.setBuildInstance(AbstractExecutable.getBuildInstance(output)); result.setExecStartTime(AbstractExecutable.getStartTime(output));
@Test public void testDiscard() throws Exception { logger.info("testDiscard"); DefaultChainedExecutable job = new DefaultChainedExecutable(); SelfStopExecutable task1 = new SelfStopExecutable(); job.addTask(task1); execMgr.addJob(job); Thread.sleep(1100); // give time to launch job/task1 waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500); execMgr.discardJob(job.getId()); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.DISCARDED, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.DISCARDED, execMgr.getOutput(task1.getId()).getState()); task1.waitForDoWork(); }
@Test public void testIllegalState() throws Exception { logger.info("testIllegalState"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); BaseTestExecutable task2 = new RunningTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()).updateJobOutput(task2.getId(), ExecutableState.RUNNING, null, null); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(task2.getId()).getState()); }