private boolean isTableInUse(String segUuid, List<String> workingJobList) { for (String jobId : workingJobList) { AbstractExecutable abstractExecutable = executableManager.getJob(jobId); String segmentId = abstractExecutable.getParam("segmentId"); if (null == segmentId) continue; return segUuid.equals(segmentId); } return false; }
public void resumeJob(String jobId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } Map<String, String> info = null; if (job instanceof DefaultChainedExecutable) { List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks(); for (AbstractExecutable task : tasks) { if (task.getStatus() == ExecutableState.ERROR || task.getStatus() == ExecutableState.STOPPED) { updateJobOutput(task.getId(), ExecutableState.READY, null, null); break; } } final long endTime = job.getEndTime(); if (endTime != 0) { long interruptTime = System.currentTimeMillis() - endTime + job.getInterruptTime(); info = Maps.newHashMap(getJobOutput(jobId).getInfo()); info.put(AbstractExecutable.INTERRUPT_TIME, Long.toString(interruptTime)); info.remove(AbstractExecutable.END_TIME); } } updateJobOutput(jobId, ExecutableState.READY, info, null); }
public void pauseJob(String jobId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } updateJobOutput(jobId, ExecutableState.STOPPED, null, null); }
private String getSegmentIdFromJobId(String jobId) { AbstractExecutable abstractExecutable = executableManager.getJob(jobId); String segmentId = abstractExecutable.getParam("segmentId"); return segmentId; }
private boolean checkSkip(String cubingJobId) { if (cubingJobId == null) return false; ExecutableManager execMgr = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); CubingJob cubingJob = (CubingJob) execMgr.getJob(cubingJobId); skipped = !cubingJob.isInMemCubing(); return skipped; }
public JobInstance getJobInstance(String uuid) { AbstractExecutable job = getExecutableManager().getJob(uuid); if (job instanceof CheckpointExecutable) { return getCheckpointJobInstance(job); } else { return getSingleJobInstance(job); } }
private boolean checkSkip(String cubingJobId) { if (cubingJobId == null) return false; ExecutableManager execMgr = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); CubingJob cubingJob = (CubingJob) execMgr.getJob(cubingJobId); skipped = cubingJob.isInMemCubing() == false; return skipped; }
protected void waitForJobStatus(String jobId, ExecutableState state, long interval) { while (true) { AbstractExecutable job = execMgr.getJob(jobId); if (job.getStatus() == state) { break; } else { try { Thread.sleep(interval); } catch (InterruptedException e) { e.printStackTrace(); } } } }
private void setAlgorithmLayer() { ExecutableManager execMgr = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); CubingJob cubingJob = (CubingJob) execMgr.getJob(this.getParam(JOB_ID)); cubingJob.setAlgorithm(CubingJob.AlgorithmEnum.LAYER); }
private boolean checkSkip(String cubingJobId, int level) { if (cubingJobId == null) return false; ExecutableManager execMgr = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); CubingJob cubingJob = (CubingJob) execMgr.getJob(cubingJobId); skipped = cubingJob.isLayerCubing() == false; if (!skipped) { skipped = (level > cuboidScheduler.getBuildLevel()); if (skipped) { logger.info("Job level: " + level + " for " + cubingJobId + "[" + cubingJobId + "] exceeds real cuboid tree levels : " + cuboidScheduler.getBuildLevel()); } } return skipped; }
public void rollbackJob(String jobId, String stepId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } if (job instanceof DefaultChainedExecutable) { List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks(); for (AbstractExecutable task : tasks) { if (task.getId().compareTo(stepId) >= 0) { logger.debug("rollback task : " + task); updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), ""); } } } if (job.getStatus() == ExecutableState.SUCCEED) { updateJobOutput(job.getId(), ExecutableState.READY, null, null); } }
private ProjectInstance getProjectByJob(JobInstance job) { AbstractExecutable executable = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()) .getJob(job.getUuid()); String projectName = ((CubingJob) executable).getProjectName(); return getProjectInstance(projectName); }
private void resumeAllRunningJobs() { for (final String id : executableManager.getAllJobIds()) { final Output output = executableManager.getOutput(id); AbstractExecutable executable = executableManager.getJob(id); if (output.getState() == ExecutableState.RUNNING && executable instanceof DefaultChainedExecutable) { try { if (!jobLock.isLocked(getLockPath(executable.getId()))) { executableManager.resumeRunningJobForce(executable.getId()); fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS); } } catch (Exception e) { logger.error("resume the job " + id + " fail in server: " + serverName, e); } } } }
public void resumeRunningJobForce(String jobId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } if (job instanceof DefaultChainedExecutable) { List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks(); for (AbstractExecutable task : tasks) { if (task.getStatus() == ExecutableState.RUNNING) { updateJobOutput(task.getId(), ExecutableState.READY, null, null); break; } } } updateJobOutput(jobId, ExecutableState.READY, null, null); }
protected void runningJobToError(String jobId) { while (true) { try { AbstractExecutable job = execMgr.getJob(jobId); ExecutableState status = job.getStatus(); if (status == ExecutableState.RUNNING) { scheduler.getFetcherRunner().setFetchFailed(true); break; } Thread.sleep(1000); } catch (Exception ex) { logger.error("", ex); } } } }
public void cancelJob(JobInstance job) throws IOException { aclEvaluate.checkProjectOperationPermission(job); if (null == job.getRelatedCube() || null == getCubeManager().getCube(job.getRelatedCube()) || null == job.getRelatedSegment()) { getExecutableManager().discardJob(job.getId()); } logger.info("Cancel job [" + job.getId() + "] trigger by " + SecurityContextHolder.getContext().getAuthentication().getName()); if (job.getStatus() == JobStatusEnum.FINISHED) { throw new IllegalStateException( "The job " + job.getId() + " has already been finished and cannot be discarded."); } if (job.getStatus() != JobStatusEnum.DISCARDED) { AbstractExecutable executable = getExecutableManager().getJob(job.getId()); if (executable instanceof CubingJob) { cancelCubingJobInner((CubingJob) executable); } else if (executable instanceof CheckpointExecutable) { cancelCheckpointJobInner((CheckpointExecutable) executable); } else { getExecutableManager().discardJob(executable.getId()); } } }
public void discardJob(String jobId) { AbstractExecutable job = getJob(jobId); if (job == null) { return; } if (job.getStatus().isFinalState()) { if (job.getStatus() != ExecutableState.DISCARDED) { logger.warn("The status of job " + jobId + " is " + job.getStatus().toString() + ". It's final state and cannot be transfer to be discarded!!!"); } else { logger.warn("The job " + jobId + " has been discarded."); } return; } if (job instanceof DefaultChainedExecutable) { List<AbstractExecutable> tasks = ((DefaultChainedExecutable) job).getTasks(); for (AbstractExecutable task : tasks) { if (!task.getStatus().isFinalState()) { updateJobOutput(task.getId(), ExecutableState.DISCARDED, null, null); } } } updateJobOutput(jobId, ExecutableState.DISCARDED, null, null); }
@Test public void testDefaultChainedExecutable() throws Exception { DefaultChainedExecutable job = new DefaultChainedExecutable(); job.addTask(new SucceedTestExecutable()); job.addTask(new SucceedTestExecutable()); service.addJob(job); assertEquals(2, job.getTasks().size()); AbstractExecutable anotherJob = service.getJob(job.getId()); assertEquals(DefaultChainedExecutable.class, anotherJob.getClass()); assertEquals(2, ((DefaultChainedExecutable) anotherJob).getTasks().size()); assertJobEqual(job, anotherJob); }
@Test public void test() throws Exception { assertNotNull(service); // all existing are broken jobs List<AbstractExecutable> existing = service.getAllExecutables(); for (AbstractExecutable exec : existing) { assertEquals("BrokenExecutable", exec.getClass().getSimpleName()); assertEquals(ExecutableState.DISCARDED, exec.getStatus()); } BaseTestExecutable executable = new SucceedTestExecutable(); executable.setParam("test1", "test1"); executable.setParam("test2", "test2"); executable.setParam("test3", "test3"); service.addJob(executable); List<AbstractExecutable> result = service.getAllExecutables(); assertEquals(existing.size() + 1, result.size()); AbstractExecutable another = service.getJob(executable.getId()); assertJobEqual(executable, another); service.updateJobOutput(executable.getId(), ExecutableState.RUNNING, null, "test output"); assertJobEqual(executable, service.getJob(executable.getId())); }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final CubeManager cubeManager = CubeManager.getInstance(context.getConfig()); final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())); final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams())); CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(segment); long sourceCount = originalSegment.getInputRecords(); long sourceSizeBytes = originalSegment.getInputRecordsSize(); CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams())); long cubeSizeBytes = cubingJob.findCubeSizeBytes(); segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams())); segment.setLastBuildTime(System.currentTimeMillis()); segment.setSizeKB(cubeSizeBytes / 1024); segment.setInputRecords(sourceCount); segment.setInputRecordsSize(sourceSizeBytes); segment.setDimensionRangeInfoMap(originalSegment.getDimensionRangeInfoMap()); try { cubeManager.promoteNewlyOptimizeSegments(cube, segment); return new ExecuteResult(); } catch (IOException e) { logger.error("fail to update cube after build", e); return ExecuteResult.createError(e); } }