@Override public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow) { jobFlow.addTask(createGCStep(intermediateTables, intermediatePaths)); } }
@Override public void addStepPhase1_MergeDictionary(DefaultChainedExecutable jobFlow) { jobFlow.addTask(createMergeOffsetStep(jobFlow.getId(), cubeSegment)); } }
@Override public void addStepPhase2_BuildDictionary(DefaultChainedExecutable jobFlow) { jobFlow.addTask(steps.createCreateHTableStep(jobFlow.getId())); }
@Override public void addStepPhase2_BuildCube(CubeSegment seg, List<CubeSegment> mergingSegments, DefaultChainedExecutable jobFlow) { jobFlow.addTask( steps.createMergeCuboidDataStep(seg, mergingSegments, jobFlow.getId(), MergeCuboidJob.class)); jobFlow.addTask(steps.createConvertCuboidToHfileStep(jobFlow.getId())); jobFlow.addTask(steps.createBulkLoadStep(jobFlow.getId())); }
@Override public void addStepPhase1_MergeDictionary(DefaultChainedExecutable jobFlow) { jobFlow.addTask(steps.createCreateHTableStep(jobFlow.getId())); }
@Override protected void addStepPhase1_DoCreateFlatTable(DefaultChainedExecutable jobFlow) { final String cubeName = CubingExecutableUtil.getCubeName(jobFlow.getParams()); final String hiveInitStatements = JoinedFlatTable.generateHiveInitStatements(flatTableDatabase); final String jobWorkingDir = getJobWorkingDir(jobFlow, hdfsWorkingDir); jobFlow.addTask(createSqoopToFlatHiveStep(jobWorkingDir, cubeName)); jobFlow.addTask(createFlatHiveTableFromFiles(hiveInitStatements, jobWorkingDir)); }
public void addCheckpointGarbageCollectionSteps(DefaultChainedExecutable jobFlow) { String jobId = jobFlow.getId(); jobFlow.addTask(createOptimizeGCStep()); List<String> toDeletePaths = new ArrayList<>(); toDeletePaths.addAll(getOptimizeHDFSPaths()); HDFSPathGarbageCollectionStep step = new HDFSPathGarbageCollectionStep(); step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION_HDFS); step.setDeletePaths(toDeletePaths); step.setJobId(jobId); jobFlow.addTask(step); }
public void addMergingGarbageCollectionSteps(DefaultChainedExecutable jobFlow) { String jobId = jobFlow.getId(); jobFlow.addTask(createMergeGCStep()); List<String> toDeletePaths = new ArrayList<>(); toDeletePaths.addAll(getMergingHDFSPaths()); toDeletePaths.add(getHFilePath(jobId)); HDFSPathGarbageCollectionStep step = new HDFSPathGarbageCollectionStep(); step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION_HDFS); step.setDeletePaths(toDeletePaths); step.setJobId(jobId); jobFlow.addTask(step); }
public void addCubingGarbageCollectionSteps(DefaultChainedExecutable jobFlow) { String jobId = jobFlow.getId(); List<String> toDeletePaths = new ArrayList<>(); toDeletePaths.add(getFactDistinctColumnsPath(jobId)); toDeletePaths.add(getHFilePath(jobId)); toDeletePaths.add(getShrunkenDictionaryPath(jobId)); HDFSPathGarbageCollectionStep step = new HDFSPathGarbageCollectionStep(); step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION_HBASE); step.setDeletePaths(toDeletePaths); step.setJobId(jobId); jobFlow.addTask(step); }
public void addOptimizeGarbageCollectionSteps(DefaultChainedExecutable jobFlow) { String jobId = jobFlow.getId(); List<String> toDeletePaths = new ArrayList<>(); toDeletePaths.add(getOptimizationRootPath(jobId)); HDFSPathGarbageCollectionStep step = new HDFSPathGarbageCollectionStep(); step.setName(ExecutableConstants.STEP_NAME_GARBAGE_COLLECTION_HDFS); step.setDeletePaths(toDeletePaths); step.setJobId(jobId); jobFlow.addTask(step); }
protected void addStepPhase1_DoMaterializeLookupTable(DefaultChainedExecutable jobFlow) { final String hiveInitStatements = JoinedFlatTable.generateHiveInitStatements(flatTableDatabase); final String jobWorkingDir = getJobWorkingDir(jobFlow, hdfsWorkingDir); AbstractExecutable task = createLookupHiveViewMaterializationStep(hiveInitStatements, jobWorkingDir, flatDesc, hiveViewIntermediateTables, jobFlow.getId()); if (task != null) { jobFlow.addTask(task); } }
protected void addStepPhase1_DoCreateFlatTable(DefaultChainedExecutable jobFlow) { final String cubeName = CubingExecutableUtil.getCubeName(jobFlow.getParams()); final String hiveInitStatements = JoinedFlatTable.generateHiveInitStatements(flatTableDatabase); final String jobWorkingDir = getJobWorkingDir(jobFlow, hdfsWorkingDir); jobFlow.addTask(createFlatHiveTableStep(hiveInitStatements, jobWorkingDir, cubeName, flatDesc)); }
@Override public void addStepPhase4_Cleanup(DefaultChainedExecutable jobFlow) { final String jobWorkingDir = getJobWorkingDir(jobFlow, hdfsWorkingDir); org.apache.kylin.source.hive.GarbageCollectionStep step = new org.apache.kylin.source.hive.GarbageCollectionStep(); step.setName(ExecutableConstants.STEP_NAME_HIVE_CLEANUP); step.setIntermediateTables(Collections.singletonList(getIntermediateTableIdentity())); step.setExternalDataPaths(Collections.singletonList(JoinedFlatTable.getTableDir(flatDesc, jobWorkingDir))); step.setHiveViewIntermediateTableIdentities(StringUtil.join(hiveViewIntermediateTables, ",")); jobFlow.addTask(step); }
private void addUpdateSnapshotQueryCacheStep(DefaultChainedExecutable jobFlow, String tableName, String snapshotID) { UpdateSnapshotCacheForQueryServersStep updateSnapshotCacheStep = new UpdateSnapshotCacheForQueryServersStep(); updateSnapshotCacheStep.setName(ExecutableConstants.STEP_NAME_LOOKUP_SNAPSHOT_CACHE_UPDATE + ":" + tableName); LookupExecutableUtil.setProjectName(cube.getProject(), updateSnapshotCacheStep.getParams()); LookupExecutableUtil.setLookupTableName(tableName, updateSnapshotCacheStep.getParams()); LookupExecutableUtil.setLookupSnapshotID(snapshotID, updateSnapshotCacheStep.getParams()); jobFlow.addTask(updateSnapshotCacheStep); }
@Test public void testDefaultChainedExecutable() throws Exception { DefaultChainedExecutable job = new DefaultChainedExecutable(); job.addTask(new SucceedTestExecutable()); job.addTask(new SucceedTestExecutable()); service.addJob(job); assertEquals(2, job.getTasks().size()); AbstractExecutable anotherJob = service.getJob(job.getId()); assertEquals(DefaultChainedExecutable.class, anotherJob.getClass()); assertEquals(2, ((DefaultChainedExecutable) anotherJob).getTasks().size()); assertJobEqual(job, anotherJob); }
@Test public void testSingleTaskJob() throws Exception { logger.info("testSingleTaskJob"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); job.addTask(task1); execMgr.addJob(job); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); }