@Override protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) { super.onExecuteFinished(result, executableContext); if (!isDiscarded() && result.succeed()) { List<? extends Executable> jobs = getTasks(); boolean allSucceed = true; for (Executable task : jobs) { final ExecutableState status = task.getStatus(); if (status != ExecutableState.SUCCEED) { allSucceed = false; } } if (allSucceed) { // Add last optimization time CubeManager cubeManager = CubeManager.getInstance(executableContext.getConfig()); CubeInstance cube = cubeManager.getCube(getCubeName()); CubeInstance copyForWrite = cube.latestCopyForWrite(); try { copyForWrite.setCuboidLastOptimized(getEndTime()); CubeUpdate cubeUpdate = new CubeUpdate(copyForWrite); cubeManager.updateCube(cubeUpdate); } catch (IOException e) { logger.error("Failed to update last optimized for " + getCubeName(), e); } } } }
private void cancelCheckpointJobInner(CheckpointExecutable checkpointExecutable) throws IOException { List<String> segmentIdList = Lists.newLinkedList(); List<String> jobIdList = Lists.newLinkedList(); jobIdList.add(checkpointExecutable.getId()); setRelatedIdList(checkpointExecutable, segmentIdList, jobIdList); CubeInstance cubeInstance = getCubeManager() .getCube(CubingExecutableUtil.getCubeName(checkpointExecutable.getParams())); if (!segmentIdList.isEmpty()) { List<CubeSegment> toRemoveSegments = Lists.newLinkedList(); for (String segmentId : segmentIdList) { final CubeSegment segment = cubeInstance.getSegmentById(segmentId); if (segment != null && segment.getStatus() != SegmentStatusEnum.READY) { toRemoveSegments.add(segment); } } CubeUpdate cubeBuilder = new CubeUpdate(cubeInstance); cubeBuilder.setToRemoveSegs(toRemoveSegments.toArray(new CubeSegment[toRemoveSegments.size()])); cubeBuilder.setCuboidsRecommend(Sets.<Long> newHashSet()); //Set recommend cuboids to be null getCubeManager().updateCube(cubeBuilder); } for (String jobId : jobIdList) { getExecutableManager().discardJob(jobId); } }
@Override public boolean apply(@Nullable CheckpointExecutable checkpointExecutable) { if (checkpointExecutable == null) { return false; } if (Strings.isEmpty(jobName)) { return true; } if (nameExactMatch) { return checkpointExecutable.getName().equalsIgnoreCase(jobName); } else { return checkpointExecutable.getName().toLowerCase(Locale.ROOT) .contains(jobName.toLowerCase(Locale.ROOT)); } } })));
public CheckpointExecutable build() { KylinConfig kylinConfig = cube.getConfig(); List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(), cube.getName()); if (projList == null || projList.size() == 0) { throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!"); } else if (projList.size() >= 2) { throw new RuntimeException("Find more than one project containing the cube " + cube.getName() + ". It does't meet the uniqueness requirement!!! "); } CheckpointExecutable checkpointJob = new CheckpointExecutable(); checkpointJob.setSubmitter(submitter); CubingExecutableUtil.setCubeName(cube.getName(), checkpointJob.getParams()); checkpointJob.setName( cube.getName() + " - OPTIMIZE CHECKPOINT - " + format.format(new Date(System.currentTimeMillis()))); checkpointJob.setDeployEnvName(kylinConfig.getDeployEnv()); checkpointJob.setProjectName(projList.get(0).getName()); // Phase 1: Update cube information checkpointJob.addTask(createUpdateCubeInfoAfterCheckpointStep()); // Phase 2: Garbage collection outputSide.addStepPhase5_Cleanup(checkpointJob); return checkpointJob; }
Output output = outputs.get(job.getId()); if (output == null) { logger.warn("job output is null."); result.setName(job.getName()); result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams())); result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams())); result.setLastModified(output.getLastModified()); result.setSubmitter(job.getSubmitter()); result.setUuid(job.getId()); result.setType(CubeBuildTypeEnum.CHECKPOINT); result.setStatus(parseToJobStatus(output.getState())); result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(), result.getExecInterruptTime()) / 1000); for (int i = 0; i < job.getTasks().size(); ++i) { AbstractExecutable task = job.getTasks().get(i); result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
for (AbstractExecutable taskForCheck : checkpointExecutable.getSubTasksForCheck()) { if (taskForCheck instanceof CubingJob) { CubingJob subCubingJob = (CubingJob) taskForCheck; + " in CheckpointExecutable " + checkpointExecutable.getName()); checkpointExecutable.getSubTasksForCheck().set(checkpointExecutable.getSubTasksForCheck().indexOf(toBeReplaced), optimizeJob); getExecutableManager().updateCheckpointJob(checkpointExecutable.getId(), checkpointExecutable.getSubTasksForCheck());
protected JobInstance getCheckpointJobInstance(AbstractExecutable job) { Message msg = MsgPicker.getMsg(); if (job == null) { return null; } if (!(job instanceof CheckpointExecutable)) { throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_JOB_TYPE(), job.getId())); } CheckpointExecutable checkpointExecutable = (CheckpointExecutable) job; Output output = checkpointExecutable.getOutput(); final JobInstance result = new JobInstance(); result.setName(job.getName()); result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams())); result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams())); result.setLastModified(job.getLastModified()); result.setSubmitter(job.getSubmitter()); result.setUuid(job.getId()); result.setType(CubeBuildTypeEnum.CHECKPOINT); result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus())); result.setBuildInstance(AbstractExecutable.getBuildInstance(output)); result.setDuration(job.getDuration() / 1000); for (int i = 0; i < checkpointExecutable.getTasks().size(); ++i) { AbstractExecutable task = checkpointExecutable.getTasks().get(i); result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId()))); } return result; }
private void setRelatedIdList(CheckpointExecutable checkpointExecutable, List<String> segmentIdList, List<String> jobIdList) { for (AbstractExecutable taskForCheck : checkpointExecutable.getSubTasksForCheck()) { jobIdList.add(taskForCheck.getId()); if (taskForCheck instanceof CubingJob) { segmentIdList.addAll(Lists .newArrayList(StringUtils.split(CubingExecutableUtil.getSegmentId(taskForCheck.getParams())))); } else if (taskForCheck instanceof CheckpointExecutable) { setRelatedIdList((CheckpointExecutable) taskForCheck, segmentIdList, jobIdList); } } }
@Override public boolean apply(CheckpointExecutable executable) { try { Output output = allOutputs.get(executable.getId()); if (output == null) { return false; } ExecutableState state = output.getState(); boolean ret = statusList.contains(state); return ret; } catch (Exception e) { throw e; } } }, new Predicate<CheckpointExecutable>() {
Preconditions.checkArgument(result instanceof CheckpointExecutable); for (ExecutablePO subTaskForCheck : tasksForCheck) { ((CheckpointExecutable) result).addTaskForCheck(parseTo(subTaskForCheck));
checkpointJob.addTaskListForCheck(optimizeJobList);
public static JobInstance parseToJobInstance(CheckpointExecutable job, Map<String, Output> outputs) { if (job == null) { logger.warn("job is null."); return null; } Output output = outputs.get(job.getId()); if (output == null) { logger.warn("job output is null."); return null; } final JobInstance result = new JobInstance(); result.setName(job.getName()); result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams())); result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams())); result.setLastModified(output.getLastModified()); result.setSubmitter(job.getSubmitter()); result.setUuid(job.getId()); result.setType(CubeBuildTypeEnum.CHECKPOINT); result.setStatus(parseToJobStatus(output.getState())); result.setExecStartTime(AbstractExecutable.getStartTime(output)); result.setExecEndTime(AbstractExecutable.getEndTime(output)); result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output)); result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(), result.getExecInterruptTime()) / 1000); for (int i = 0; i < job.getTasks().size(); ++i) { AbstractExecutable task = job.getTasks().get(i); result.addStep(parseToJobStep(task, i, outputs.get(task.getId()))); } return result; }
public CheckpointExecutable build() { KylinConfig kylinConfig = cube.getConfig(); List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(), cube.getName()); if (projList == null || projList.size() == 0) { throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!"); } else if (projList.size() >= 2) { throw new RuntimeException("Find more than one project containing the cube " + cube.getName() + ". It does't meet the uniqueness requirement!!! "); } CheckpointExecutable checkpointJob = new CheckpointExecutable(); checkpointJob.setSubmitter(submitter); CubingExecutableUtil.setCubeName(cube.getName(), checkpointJob.getParams()); checkpointJob.setName( cube.getName() + " - OPTIMIZE CHECKPOINT - " + format.format(new Date(System.currentTimeMillis()))); checkpointJob.setDeployEnvName(kylinConfig.getDeployEnv()); checkpointJob.setProjectName(projList.get(0).getName()); // Phase 1: Update cube information checkpointJob.addTask(createUpdateCubeInfoAfterCheckpointStep()); // Phase 2: Garbage collection outputSide.addStepPhase5_Cleanup(checkpointJob); return checkpointJob; }
private static ExecutablePO parse(AbstractExecutable executable) { ExecutablePO result = new ExecutablePO(); result.setName(executable.getName()); result.setUuid(executable.getId()); result.setType(executable.getClass().getName()); result.setParams(executable.getParams()); if (executable instanceof ChainedExecutable) { List<ExecutablePO> tasks = Lists.newArrayList(); for (AbstractExecutable task : ((ChainedExecutable) executable).getTasks()) { tasks.add(parse(task)); } result.setTasks(tasks); } if (executable instanceof CheckpointExecutable) { List<ExecutablePO> tasksForCheck = Lists.newArrayList(); for (AbstractExecutable taskForCheck : ((CheckpointExecutable) executable).getSubTasksForCheck()) { tasksForCheck.add(parse(taskForCheck)); } result.setTasksForCheck(tasksForCheck); } return result; }
@Override public boolean apply(CheckpointExecutable executable) { try { ExecutableOutputPO executableOutputPO = allExecutableOutputPO .get(executable.getId()); ExecutableState state = ExecutableState.valueOf(executableOutputPO.getStatus()); return statusList.contains(state); } catch (Exception e) { throw e; } } }, new Predicate<CheckpointExecutable>() {
Preconditions.checkArgument(result instanceof CheckpointExecutable); for (ExecutablePO subTaskForCheck : tasksForCheck) { ((CheckpointExecutable) result).addTaskForCheck(parseTo(subTaskForCheck));
@Override protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) { super.onExecuteFinished(result, executableContext); if (!isDiscarded() && result.succeed()) { List<? extends Executable> jobs = getTasks(); boolean allSucceed = true; for (Executable task : jobs) { final ExecutableState status = task.getStatus(); if (status != ExecutableState.SUCCEED) { allSucceed = false; } } if (allSucceed) { // Add last optimization time CubeManager cubeManager = CubeManager.getInstance(executableContext.getConfig()); CubeInstance cube = cubeManager.getCube(getCubeName()); CubeInstance copyForWrite = cube.latestCopyForWrite(); try { copyForWrite.setCuboidLastOptimized(getEndTime()); CubeUpdate cubeUpdate = new CubeUpdate(copyForWrite); cubeManager.updateCube(cubeUpdate); } catch (IOException e) { logger.error("Failed to update last optimized for " + getCubeName(), e); } } } }
private static ExecutablePO parse(AbstractExecutable executable) { ExecutablePO result = new ExecutablePO(); result.setName(executable.getName()); result.setUuid(executable.getId()); result.setType(executable.getClass().getName()); result.setParams(executable.getParams()); if (executable instanceof ChainedExecutable) { List<ExecutablePO> tasks = Lists.newArrayList(); for (AbstractExecutable task : ((ChainedExecutable) executable).getTasks()) { tasks.add(parse(task)); } result.setTasks(tasks); } if (executable instanceof CheckpointExecutable) { List<ExecutablePO> tasksForCheck = Lists.newArrayList(); for (AbstractExecutable taskForCheck : ((CheckpointExecutable) executable).getSubTasksForCheck()) { tasksForCheck.add(parse(taskForCheck)); } result.setTasksForCheck(tasksForCheck); } return result; }
@Override public boolean apply(@Nullable CheckpointExecutable checkpointExecutable) { if (checkpointExecutable == null) { return false; } if (Strings.isEmpty(jobName)) { return true; } if (nameExactMatch) { return checkpointExecutable.getName().equalsIgnoreCase(jobName); } else { return checkpointExecutable.getName().toLowerCase(Locale.ROOT) .contains(jobName.toLowerCase(Locale.ROOT)); } } })));