- Add the Codota plugin to your IDE and get smart completions
private void myMethod () {}
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { KylinConfig config = context.getConfig(); StringBuffer output = new StringBuffer(); try { output.append(cleanUpIntermediateFlatTable(config)); // don't drop view to avoid concurrent issue //output.append(cleanUpHiveViewIntermediateTable(config)); } catch (IOException e) { logger.error("job:" + getId() + " execute finished with exception", e); return ExecuteResult.createError(e); } return new ExecuteResult(ExecuteResult.State.SUCCEED, output.toString()); }
protected void handleMetadataPersistException(ExecutableContext context, Throwable exception) { final String[] adminDls = context.getConfig().getAdminDls(); if (adminDls == null || adminDls.length < 1) { logger.warn(NO_NEED_TO_SEND_EMAIL_USER_LIST_IS_EMPTY); return; } List<String> users = Lists.newArrayList(adminDls); Map<String, Object> dataMap = Maps.newHashMap(); dataMap.put("job_name", getName()); dataMap.put("env_name", context.getConfig().getDeployEnv()); dataMap.put(SUBMITTER, StringUtil.noBlank(getSubmitter(), "missing submitter")); dataMap.put("job_engine", MailNotificationUtil.getLocalHostName()); dataMap.put("error_log", Matcher.quoteReplacement(StringUtil.noBlank(exception.getMessage(), "no error message"))); String content = MailNotificationUtil.getMailContent(MailNotificationUtil.METADATA_PERSIST_FAIL, dataMap); String title = MailNotificationUtil.getMailTitle("METADATA PERSIST", "FAIL", context.getConfig().getDeployEnv()); new MailService(context.getConfig()).sendMail(users, title, content); }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { try { config = new JobEngineConfig(context.getConfig()); List<String> toDeletePaths = getDeletePaths(); dropHdfsPathOnCluster(toDeletePaths, HadoopUtil.getWorkingFileSystem()); if (StringUtils.isNotEmpty(context.getConfig().getHBaseClusterFs())) { dropHdfsPathOnCluster(toDeletePaths, FileSystem.get(HBaseConnection.getCurrentHBaseConfiguration())); } } catch (IOException e) { logger.error("job:" + getId() + " execute finished with exception", e); output.append("\n").append(e.getLocalizedMessage()); return new ExecuteResult(ExecuteResult.State.ERROR, output.toString(), e); } return new ExecuteResult(ExecuteResult.State.SUCCEED, output.toString()); }
public static CubeSegment findSegment(ExecutableContext context, String cubeName, String segmentId) { final CubeManager mgr = CubeManager.getInstance(context.getConfig()); final CubeInstance cube = mgr.getCube(cubeName); if (cube == null) { String cubeList = StringUtils .join(Iterables.transform(mgr.listAllCubes(), new Function<CubeInstance, String>() { @Nullable @Override public String apply(@Nullable CubeInstance input) { return input.getName(); } }).iterator(), ","); throw new IllegalStateException("target cube name: " + cubeName + " cube list: " + cubeList); } final CubeSegment newSegment = cube.getSegmentById(segmentId); if (newSegment == null) { String segmentList = StringUtils .join(Iterables.transform(cube.getSegments(), new Function<CubeSegment, String>() { @Nullable @Override public String apply(@Nullable CubeSegment input) { return input.getUuid(); } }).iterator(), ","); throw new IllegalStateException("target segment id: " + segmentId + " segment list: " + segmentList); } return newSegment; }
private void onExecuteFinishedWithRetry(ExecuteResult result, ExecutableContext executableContext) throws ExecuteException { Throwable exception; int nRetry = 0; do { nRetry++; exception = null; try { onExecuteFinished(result, executableContext); } catch (Exception e) { logger.error(nRetry + "th retries for onExecuteFinished fails due to {}", e); if (isMetaDataPersistException(e, 5)) { exception = e; try { Thread.sleep(1000L * (long) Math.pow(4, nRetry)); } catch (InterruptedException e1) { throw new IllegalStateException(e1); } } else { throw e; } } } while (exception != null && nRetry <= executableContext.getConfig().getJobMetadataPersistRetry()); if (exception != null) { handleMetadataPersistException(executableContext, exception); throw new ExecuteException(exception); } }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final CubeManager cubeManager = CubeManager.getInstance(context.getConfig()); final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())); Set<Long> recommendCuboids = cube.getCuboidsRecommend(); try { List<CubeSegment> newSegments = cube.getSegments(SegmentStatusEnum.READY_PENDING); Map<Long, Long> recommendCuboidsWithStats = CuboidStatsReaderUtil .readCuboidStatsFromSegments(recommendCuboids, newSegments); if (recommendCuboidsWithStats == null) { throw new RuntimeException("Fail to get statistics info for recommended cuboids after optimization!!!"); } cubeManager.promoteCheckpointOptimizeSegments(cube, recommendCuboidsWithStats, newSegments.toArray(new CubeSegment[newSegments.size()])); return new ExecuteResult(); } catch (Exception e) { logger.error("fail to update cube after build", e); return ExecuteResult.createError(e); } }
overwriteJobConf(conf, executableContext.getConfig(), getMapReduceParams().trim().split("\\s+")); Job job = new Cluster(conf).getJob(JobID.forName(mrJobId)); if (job == null || job.getJobState() == JobStatus.State.FAILED) {
@Override protected Pair<String, String> formatNotifications(ExecutableContext context, ExecutableState state) { CubeInstance cubeInstance = CubeManager.getInstance(context.getConfig()) .getCube(CubingExecutableUtil.getCubeName(this.getParams())); final Output output = getManager().getOutput(getId());
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final CubeManager mgr = CubeManager.getInstance(context.getConfig()); final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams())); final CubeSegment optimizeSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final CubeManager mgr = CubeManager.getInstance(context.getConfig()); final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite(); final CubeSegment optimizeSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams())); CubeSegment oldSegment = optimizeSegment.getCubeInstance().getOriginalSegmentToOptimize(optimizeSegment); Preconditions.checkNotNull(oldSegment, "cannot find the original segment to be optimized by " + optimizeSegment); // --- Copy dictionary optimizeSegment.getDictionaries().putAll(oldSegment.getDictionaries()); optimizeSegment.getSnapshots().putAll(oldSegment.getSnapshots()); optimizeSegment.getRowkeyStats().addAll(oldSegment.getRowkeyStats()); try { CubeUpdate cubeBuilder = new CubeUpdate(cube); cubeBuilder.setToUpdateSegs(optimizeSegment); mgr.updateCube(cubeBuilder); } catch (IOException e) { logger.error("fail to merge dictionary or lookup snapshots", e); return ExecuteResult.createError(e); } return new ExecuteResult(); } }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { KylinConfig kylinConfig = context.getConfig(); CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final CubeManager cubeMgr = CubeManager.getInstance(context.getConfig()); final DictionaryManager dictMgrHdfs; final DictionaryManager dictMgrHbase;
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { try { logger.info("executing:" + getCmd()); final PatternedLogger patternedLogger = new PatternedLogger(logger); final Pair<Integer, String> result = context.getConfig().getCliCommandExecutor().execute(getCmd(), patternedLogger); getManager().addJobInfo(getId(), patternedLogger.getInfo()); return result.getFirst() == 0 ? new ExecuteResult(ExecuteResult.State.SUCCEED, result.getSecond()) : ExecuteResult.createFailed(new ShellException(result.getSecond())); } catch (IOException e) { logger.error("job:" + getId() + " execute finished with exception", e); return ExecuteResult.createError(e); } }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final CubeManager mgr = CubeManager.getInstance(context.getConfig()); final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams())); final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams())); final List<CubeSegment> mergingSegments = getMergingSegments(cube); KylinConfig conf = cube.getConfig(); Collections.sort(mergingSegments); try { checkLookupSnapshotsMustIncremental(mergingSegments); // work on copy instead of cached objects CubeInstance cubeCopy = cube.latestCopyForWrite(); CubeSegment newSegCopy = cubeCopy.getSegmentById(newSegment.getUuid()); makeDictForNewSegment(conf, cubeCopy, newSegCopy, mergingSegments); makeSnapshotForNewSegment(cubeCopy, newSegCopy, mergingSegments); CubeUpdate update = new CubeUpdate(cubeCopy); update.setToUpdateSegs(newSegCopy); mgr.updateCube(update); return ExecuteResult.createSucceed(); } catch (IOException e) { logger.error("fail to merge dictionary or lookup snapshots", e); return ExecuteResult.createError(e); } }
@Override protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) { super.onExecuteFinished(result, executableContext); if (!isDiscarded() && result.succeed()) { List<? extends Executable> jobs = getTasks(); boolean allSucceed = true; for (Executable task : jobs) { final ExecutableState status = task.getStatus(); if (status != ExecutableState.SUCCEED) { allSucceed = false; } } if (allSucceed) { // Add last optimization time CubeManager cubeManager = CubeManager.getInstance(executableContext.getConfig()); CubeInstance cube = cubeManager.getCube(getCubeName()); CubeInstance copyForWrite = cube.latestCopyForWrite(); try { copyForWrite.setCuboidLastOptimized(getEndTime()); CubeUpdate cubeUpdate = new CubeUpdate(copyForWrite); cubeManager.updateCube(cubeUpdate); } catch (IOException e) { logger.error("Failed to update last optimized for " + getCubeName(), e); } } } }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { KylinConfig kylinConfig = context.getConfig(); CubeManager cubeManager = CubeManager.getInstance(kylinConfig); TableMetadataManager metaMgr = TableMetadataManager.getInstance(kylinConfig); SnapshotManager snapshotMgr = SnapshotManager.getInstance(kylinConfig); CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams())); List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams()); String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams()); CubeDesc cubeDesc = cube.getDescriptor(); try { TableDesc tableDesc = metaMgr.getTableDesc(lookupTableName, cube.getProject()); IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null); logger.info("take snapshot for table:" + lookupTableName); SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cube.getConfig()); logger.info("update snapshot path to cube metadata"); if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) { LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName, snapshot.getResourcePath()); } else { LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName, snapshot.getResourcePath()); } return new ExecuteResult(); } catch (IOException e) { logger.error("fail to build snapshot for:" + lookupTableName, e); return ExecuteResult.createError(e); } }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final CubeManager cubeManager = CubeManager.getInstance(context.getConfig()); final CubeInstance cubeCopy = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite(); final String segmentId = CubingExecutableUtil.getSegmentId(this.getParams()); final CubeSegment segCopy = cubeCopy.getSegmentById(segmentId); Preconditions.checkNotNull(segCopy, "Cube segment '" + segmentId + "' not found."); Segments<CubeSegment> mergingSegs = cubeCopy.getMergingSegments(segCopy); Preconditions.checkArgument(mergingSegs.size() > 0, "Merging segment not exist."); Collections.sort(mergingSegs); final CubeSegment first = mergingSegs.get(0); final CubeSegment last = mergingSegs.get(mergingSegs.size() - 1); segCopy.setSegRange(new SegmentRange(first.getSegRange().start, last.getSegRange().end)); segCopy.setSourcePartitionOffsetStart(first.getSourcePartitionOffsetStart()); segCopy.setSourcePartitionOffsetEnd(last.getSourcePartitionOffsetEnd()); segCopy.setTSRange(new TSRange(mergingSegs.getTSStart(), mergingSegs.getTSEnd())); CubeUpdate update = new CubeUpdate(cubeCopy); update.setToUpdateSegs(segCopy); try { cubeManager.updateCube(update); return ExecuteResult.createSucceed(); } catch (IOException e) { logger.error("fail to update cube segment offset", e); return ExecuteResult.createError(e); } }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final CubeManager cubeManager = CubeManager.getInstance(context.getConfig()); final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())); final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams())); CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(segment); long sourceCount = originalSegment.getInputRecords(); long sourceSizeBytes = originalSegment.getInputRecordsSize(); CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams())); long cubeSizeBytes = cubingJob.findCubeSizeBytes(); segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams())); segment.setLastBuildTime(System.currentTimeMillis()); segment.setSizeKB(cubeSizeBytes / 1024); segment.setInputRecords(sourceCount); segment.setInputRecordsSize(sourceSizeBytes); segment.setDimensionRangeInfoMap(originalSegment.getDimensionRangeInfoMap()); try { cubeManager.promoteNewlyOptimizeSegments(cube, segment); return new ExecuteResult(); } catch (IOException e) { logger.error("fail to update cube after build", e); return ExecuteResult.createError(e); } }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final CubeManager cubeManager = CubeManager.getInstance(context.getConfig()); final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())) .latestCopyForWrite(); final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams())); CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams())); long sourceCount = cubingJob.findSourceRecordCount(); long sourceSizeBytes = cubingJob.findSourceSizeBytes(); long cubeSizeBytes = cubingJob.findCubeSizeBytes(); segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams())); segment.setLastBuildTime(System.currentTimeMillis()); segment.setSizeKB(cubeSizeBytes / 1024); segment.setInputRecords(sourceCount); segment.setInputRecordsSize(sourceSizeBytes); try { saveExtSnapshotIfNeeded(cubeManager, cube, segment); updateSegment(segment); cubeManager.promoteNewlyBuiltSegments(cube, segment); return new ExecuteResult(); } catch (IOException e) { logger.error("fail to update cube after build", e); return ExecuteResult.createError(e); } }
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { final CubeManager cubeManager = CubeManager.getInstance(context.getConfig()); final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite();