protected StorageCleanupJob(KylinConfig config, FileSystem defaultFs, FileSystem hbaseFs) { this.config = config; this.defaultFs = defaultFs; this.hbaseFs = hbaseFs; this.executableManager = ExecutableManager.getInstance(config); }
protected ExecutableManager getManager() { return ExecutableManager.getInstance(config); }
public ExecutableManager getExecutableManager() { return ExecutableManager.getInstance(jobEngineConfig.getConfig()); }
public ExecutableManager getExecutableManager() { return ExecutableManager.getInstance(getConfig()); }
ExecutableManager getExecutableManger() { return ExecutableManager.getInstance(jobEngineConfig.getConfig()); } }
public CubeBuildingCLI() { options = new Options(); options.addOption(OPTION_CUBE); options.addOption(OPTION_BUILD_TYPE); options.addOption(OPTION_TIME_START); options.addOption(OPTION_TIME_END); kylinConfig = KylinConfig.getInstanceFromEnv(); cubeManager = CubeManager.getInstance(kylinConfig); executableManager = ExecutableManager.getInstance(kylinConfig); }
public JobInstanceExtractor() { config = KylinConfig.getInstanceFromEnv(); executableManager = ExecutableManager.getInstance(config); projectManager = ProjectManager.getInstance(config); packageType = "jobInstances"; options.addOption(OPTION_PROJECT); options.addOption(OPTION_CUBE); options.addOption(OPTION_PERIOD); }
private boolean checkSkip(String cubingJobId) { if (cubingJobId == null) return false; ExecutableManager execMgr = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); CubingJob cubingJob = (CubingJob) execMgr.getJob(cubingJobId); skipped = !cubingJob.isInMemCubing(); return skipped; }
private boolean checkSkip(String cubingJobId) { if (cubingJobId == null) return false; ExecutableManager execMgr = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); CubingJob cubingJob = (CubingJob) execMgr.getJob(cubingJobId); skipped = cubingJob.isInMemCubing() == false; return skipped; }
private void setAlgorithmLayer() { ExecutableManager execMgr = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); CubingJob cubingJob = (CubingJob) execMgr.getJob(this.getParam(JOB_ID)); cubingJob.setAlgorithm(CubingJob.AlgorithmEnum.LAYER); }
private boolean checkSkip(String cubingJobId, int level) { if (cubingJobId == null) return false; ExecutableManager execMgr = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); CubingJob cubingJob = (CubingJob) execMgr.getJob(cubingJobId); skipped = cubingJob.isLayerCubing() == false; if (!skipped) { skipped = (level > cuboidScheduler.getBuildLevel()); if (skipped) { logger.info("Job level: " + level + " for " + cubingJobId + "[" + cubingJobId + "] exceeds real cuboid tree levels : " + cuboidScheduler.getBuildLevel()); } } return skipped; }
private ProjectInstance getProjectByJob(JobInstance job) { AbstractExecutable executable = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()) .getJob(job.getUuid()); String projectName = ((CubingJob) executable).getProjectName(); return getProjectInstance(projectName); }
this.serverName = this.jobLock.getClient(); // the lock's client string contains node name of this server executableManager = ExecutableManager.getInstance(jobEngineConfig.getConfig());
@Override public void run() { try (SetThreadName ignored = new SetThreadName("Scheduler %s Job %s", System.identityHashCode(DefaultScheduler.this), executable.getId())) { executable.execute(context); } catch (ExecuteException e) { logger.error("ExecuteException job:" + executable.getId(), e); } catch (Exception e) { if (AbstractExecutable.isMetaDataPersistException(e, 5)) { // Job fail due to PersistException ExecutableManager.getInstance(jobEngineConfig.getConfig()) .forceKillJobWithRetry(executable.getId()); } logger.error("unknown error execute job:" + executable.getId(), e); } finally { context.removeRunningJob(executable); } // trigger the next step asap fetcherPool.schedule(fetcher, 0, TimeUnit.SECONDS); } }
ExecutableManager execMgr = ExecutableManager.getInstance(kylinConfig); DefaultChainedExecutable job = (DefaultChainedExecutable) execMgr.getJob(cubingJobID);
@Before public void setup() throws Exception { System.setProperty("kylin.job.scheduler.poll-interval-second", "1"); createTestMetadata(); execMgr = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); startScheduler(); }
@Before public void setup() throws Exception { createTestMetadata("../server/src/test/resources/ut_meta/broken_executable"); service = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); }
public void calculateCardinalityIfNotPresent(String[] tables, String submitter, String prj) throws Exception { // calculate cardinality for Hive source ProjectInstance projectInstance = getProjectManager().getProject(prj); if (projectInstance == null || projectInstance.getSourceType() != ISourceAware.ID_HIVE){ return; } TableMetadataManager metaMgr = getTableManager(); ExecutableManager exeMgt = ExecutableManager.getInstance(getConfig()); for (String table : tables) { TableExtDesc tableExtDesc = metaMgr.getTableExt(table, prj); String jobID = tableExtDesc.getJodID(); if (null == jobID || ExecutableState.RUNNING != exeMgt.getOutput(jobID).getState()) { calculateCardinality(table, submitter, prj); } } }
@Test public void testIllegalState() throws Exception { logger.info("testIllegalState"); DefaultChainedExecutable job = new DefaultChainedExecutable(); BaseTestExecutable task1 = new SucceedTestExecutable(); BaseTestExecutable task2 = new RunningTestExecutable(); job.addTask(task1); job.addTask(task2); execMgr.addJob(job); ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()).updateJobOutput(task2.getId(), ExecutableState.RUNNING, null, null); waitForJobFinish(job.getId(), 10000); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(job.getId()).getState()); Assert.assertEquals(ExecutableState.SUCCEED, execMgr.getOutput(task1.getId()).getState()); Assert.assertEquals(ExecutableState.ERROR, execMgr.getOutput(task2.getId()).getState()); }
private boolean checkSkip(String cubingJobId) { if (cubingJobId == null) return false; ExecutableManager execMgr = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv()); CubingJob cubingJob = (CubingJob) execMgr.getJob(cubingJobId); skipped = !cubingJob.isInMemCubing(); return skipped; }