@Override public void nextTuple() { LOG.info("start to fetch job list"); try { List<AppInfo> apps = rmResourceFetcher.getResource(Constants.ResourceType.RUNNING_MR_JOB); if (apps == null) { apps = new ArrayList<>(); } handleApps(apps, true); long fetchTime = Calendar.getInstance().getTimeInMillis(); if (fetchTime - this.lastFinishAppTime > 60000L) { apps = rmResourceFetcher.getResource(Constants.ResourceType.COMPLETE_MR_JOB, Long.toString(this.lastFinishAppTime)); if (apps == null) { apps = new ArrayList<>(); } handleApps(apps, false); this.lastFinishAppTime = fetchTime; this.runningJobManager.updateLastFinishTime(partitionId, fetchTime); } } catch (Exception e) { LOG.warn("exception found {}", e); } finally { //need to be configured Utils.sleep(60); } }
@Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { this.rmResourceFetcher = new RMResourceFetcher(crawlConfig.endPointConfig.RMBasePaths); this.partitionId = calculatePartitionId(context); // sanity verify 0<=partitionId<=numTotalPartitions-1 if (partitionId < 0 || partitionId > crawlConfig.controlConfig.numTotalPartitions) { throw new IllegalStateException("partitionId should be less than numTotalPartitions with partitionId " + partitionId + " and numTotalPartitions " + crawlConfig.controlConfig.numTotalPartitions); } Class<? extends JobIdPartitioner> partitionerCls = crawlConfig.controlConfig.partitionerCls; try { this.jobFilter = new JobIdFilterByPartition(partitionerCls.newInstance(), crawlConfig.controlConfig.numTotalPartitions, partitionId); } catch (Exception e) { LOG.error("failing instantiating job partitioner class " + partitionerCls.getCanonicalName()); throw new IllegalStateException(e); } this.collector = collector; this.runningJobManager = new RunningJobManager(crawlConfig.zkStateConfig.zkQuorum, crawlConfig.zkStateConfig.zkSessionTimeoutMs, crawlConfig.zkStateConfig.zkRetryTimes, crawlConfig.zkStateConfig.zkRetryInterval, crawlConfig.zkStateConfig.zkRoot, crawlConfig.zkStateConfig.zkLockPath); this.lastFinishAppTime = this.runningJobManager.recoverLastFinishedTime(partitionId); if (this.lastFinishAppTime == 0L) { this.lastFinishAppTime = Calendar.getInstance().getTimeInMillis() - 24 * 60 * 60000L;//one day ago this.runningJobManager.updateLastFinishTime(partitionId, this.lastFinishAppTime); } }