private void mergeCubeSegment(String cubeName) { CubeInstance cube = getCubeManager().getCube(cubeName); if (!cube.needAutoMerge()) return; synchronized (CubeService.class) { try { cube = getCubeManager().getCube(cubeName); SegmentRange offsets = cube.autoMergeCubeSegments(); if (offsets != null) { CubeSegment newSeg = getCubeManager().mergeSegments(cube, null, offsets, true); logger.debug("Will submit merge job on " + newSeg); DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(newSeg, "SYSTEM"); getExecutableManager().addJob(job); } else { logger.debug("Not ready for merge on cube " + cubeName); } } catch (IOException e) { logger.error("Failed to auto merge cube " + cubeName, e); } } }
for (CubeSegment optimizeSegment : optimizeSegments) { DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, submitter); getExecutableManager().addJob(optimizeJob); checkpointJob.addTaskListForCheck(optimizeJobList); getExecutableManager().addJob(checkpointJob);
public JobInstance submitLookupSnapshotJob(CubeInstance cube, String lookupTable, List<String> segmentIDs, String submitter) throws IOException { Message msg = MsgPicker.getMsg(); TableDesc tableDesc = getTableManager().getTableDesc(lookupTable, cube.getProject()); if (tableDesc.isView()) { throw new BadRequestException( String.format(Locale.ROOT, msg.getREBUILD_SNAPSHOT_OF_VIEW(), tableDesc.getName())); } LookupSnapshotBuildJob job = new LookupSnapshotJobBuilder(cube, lookupTable, segmentIDs, submitter).build(); getExecutableManager().addJob(job); JobInstance jobInstance = getLookupSnapshotBuildJobInstance(job); return jobInstance; }
private void submitJob(CubeInstance cube, TSRange tsRange, CubeBuildTypeEnum buildType, boolean forceMergeEmptySeg, String submitter) throws IOException, JobException { checkCubeDescSignature(cube); DefaultChainedExecutable job; if (buildType == CubeBuildTypeEnum.BUILD) { CubeSegment newSeg = cubeManager.appendSegment(cube, tsRange); job = EngineFactory.createBatchCubingJob(newSeg, submitter); } else if (buildType == CubeBuildTypeEnum.MERGE) { CubeSegment newSeg = cubeManager.mergeSegments(cube, tsRange, null, forceMergeEmptySeg); job = EngineFactory.createBatchMergeJob(newSeg, submitter); } else if (buildType == CubeBuildTypeEnum.REFRESH) { CubeSegment refreshSeg = cubeManager.refreshSegment(cube, tsRange, null); job = EngineFactory.createBatchCubingJob(refreshSeg, submitter); } else { throw new JobException("invalid build type:" + buildType); } executableManager.addJob(job); }
getExecutableManager().addJob(optimizeJob);
@Test public void testSchedulerStop() throws Exception { logger.info("testSchedulerStop"); thrown.expect(RuntimeException.class); thrown.expectMessage("too long wait time"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new FiveSecondSucceedTestExecutable(); job.addTask(task1); execMgr.addJob(job); //sleep 3s to make sure SucceedTestExecutable is running Thread.sleep(3000); //scheduler failed due to some reason scheduler.shutdown(); waitForJobFinish(job.getId(), 6000); }
@Test(expected = IllegalStateTranferException.class) public void testInvalidStateTransfer() { SucceedTestExecutable job = new SucceedTestExecutable(); service.addJob(job); service.updateJobOutput(job.getId(), ExecutableState.ERROR, null, null); service.updateJobOutput(job.getId(), ExecutableState.STOPPED, null, null); }
@Test public void testDefaultChainedExecutable() throws Exception { DefaultChainedExecutable job = new DefaultChainedExecutable(); job.addTask(new SucceedTestExecutable()); job.addTask(new SucceedTestExecutable()); service.addJob(job); assertEquals(2, job.getTasks().size()); AbstractExecutable anotherJob = service.getJob(job.getId()); assertEquals(DefaultChainedExecutable.class, anotherJob.getClass()); assertEquals(2, ((DefaultChainedExecutable) anotherJob).getTasks().size()); assertJobEqual(job, anotherJob); }
@Test public void testMetaStoreRecover() throws Exception { logger.info("testMetaStoreRecover"); NoErrorStatusExecutable job = new NoErrorStatusExecutable(); ErrorTestExecutable task = new ErrorTestExecutable(); job.addTask(task); execMgr.addJob(job); Thread.sleep(2500); runningJobToError(job.getId()); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); }
getExecutableManager().addJob(job);
@Test public void test() throws Exception { assertNotNull(service); // all existing are broken jobs List<AbstractExecutable> existing = service.getAllExecutables(); for (AbstractExecutable exec : existing) { assertEquals("BrokenExecutable", exec.getClass().getSimpleName()); assertEquals(ExecutableState.DISCARDED, exec.getStatus()); } BaseTestExecutable executable = new SucceedTestExecutable(); executable.setParam("test1", "test1"); executable.setParam("test2", "test2"); executable.setParam("test3", "test3"); service.addJob(executable); List<AbstractExecutable> result = service.getAllExecutables(); assertEquals(existing.size() + 1, result.size()); AbstractExecutable another = service.getJob(executable.getId()); assertJobEqual(executable, another); service.updateJobOutput(executable.getId(), ExecutableState.RUNNING, null, "test output"); assertJobEqual(executable, service.getJob(executable.getId())); }
@Test public void testSingleTaskJob() throws Exception { logger.info("testSingleTaskJob"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); job.addTask(task1); execMgr.addJob(job); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); }
@Test public void testSchedulerRestart() throws Exception { logger.info("testSchedulerRestart"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new FiveSecondSucceedTestExecutable(); job.addTask(task1); execMgr.addJob(job); //sleep 3s to make sure SucceedTestExecutable is running Thread.sleep(3000); //scheduler failed due to some reason scheduler.shutdown(); //restart startScheduler(); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); }
@Test public void testValidStateTransfer() throws Exception { SucceedTestExecutable job = new SucceedTestExecutable(); String id = job.getId(); service.addJob(job); service.updateJobOutput(id, ExecutableState.RUNNING, null, null); service.updateJobOutput(id, ExecutableState.ERROR, null, null); service.updateJobOutput(id, ExecutableState.READY, null, null); service.updateJobOutput(id, ExecutableState.RUNNING, null, null); service.updateJobOutput(id, ExecutableState.READY, null, null); service.updateJobOutput(id, ExecutableState.RUNNING, null, null); service.updateJobOutput(id, ExecutableState.SUCCEED, null, null); }
@Test public void testDiscard() throws Exception { logger.info("testDiscard"); DefaultChainedExecutable job = new DefaultChainedExecutable(); SelfStopExecutable task1 = new SelfStopExecutable(); job.addTask(task1); execMgr.addJob(job); Thread.sleep(1100); // give time to launch job/task1 waitForJobStatus(job.getId(), ExecutableState.RUNNING, 500); execMgr.discardJob(job.getId()); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.DISCARDED, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.DISCARDED, execMgr.getOutput(task1.getId()).getState()); task1.waitForDoWork(); }
getTableManager().saveTableExt(tableExt, prj); getExecutableManager().addJob(job);
@Test public void testSucceedAndError() throws Exception { logger.info("testSucceedAndError"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new ErrorTestExecutable(); BaseTestExecutable task2 = new SucceedTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.READY, execMgr.getOutput(task2.getId()).getState()); }
@Test public void testSucceed() throws Exception { logger.info("testSucceed"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); BaseTestExecutable task2 = new SucceedTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task2.getId()).getState()); }
@Test public void testSucceedAndFailed() throws Exception { logger.info("testSucceedAndFailed"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); BaseTestExecutable task2 = new FailedTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(task2.getId()).getState()); }
@Test public void testIllegalState() throws Exception { logger.info("testIllegalState"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); BaseTestExecutable task2 = new RunningTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()).updateJobOutput(task2.getId(), ExecutableState.RUNNING, null, null); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(task2.getId()).getState()); }