/** * Returns all jobs tagged with the given tag that have been started after the * given timestamp. Returned jobIds are MapReduce JobIds. */ @Override public Set<String> getJobs(String tag, long timestamp) { Set<ApplicationId> childYarnJobs = getYarnChildJobs(tag, timestamp); Set<String> childJobs = new HashSet<String>(); for(ApplicationId id : childYarnJobs) { // Convert to a MapReduce job id String childJobId = TypeConverter.fromYarn(id).toString(); childJobs.add(childJobId); } return childJobs; }
@Override public GetCountersResponse getCounters(GetCountersRequest request) throws IOException { JobId jobId = request.getJobId(); Job job = verifyAndGetJob(jobId, true); GetCountersResponse response = recordFactory.newRecordInstance(GetCountersResponse.class); response.setCounters(TypeConverter.toYarn(job.getAllCounters())); return response; }
public JobHistoryCopyService(ApplicationAttemptId applicationAttemptId, EventHandler handler) { super("JobHistoryCopyService"); this.applicationAttemptId = applicationAttemptId; this.jobId = TypeConverter.toYarn( TypeConverter.fromYarn(applicationAttemptId.getApplicationId())); this.handler = handler; }
public static QueueInfo fromYarn(org.apache.hadoop.yarn.api.records.QueueInfo queueInfo, Configuration conf) { QueueInfo toReturn = new QueueInfo(queueInfo.getQueueName(), "Capacity: " + queueInfo.getCapacity() * 100 + ", MaximumCapacity: " + (queueInfo.getMaximumCapacity() < 0 ? "UNDEFINED" : queueInfo.getMaximumCapacity() * 100) + ", CurrentCapacity: " + queueInfo.getCurrentCapacity() * 100, fromYarn(queueInfo.getQueueState()), TypeConverter.fromYarnApps(queueInfo.getApplications(), conf)); List<QueueInfo> childQueues = new ArrayList<QueueInfo>(); for(org.apache.hadoop.yarn.api.records.QueueInfo childQueue : queueInfo.getChildQueues()) { childQueues.add(fromYarn(childQueue, conf)); } toReturn.setQueueChildren(childQueues); return toReturn; }
public JobStatus[] getAllJobs() throws IOException, InterruptedException { try { Set<String> appTypes = new HashSet<String>(1); appTypes.add(MRJobConfig.MR_APPLICATION_TYPE); EnumSet<YarnApplicationState> appStates = EnumSet.noneOf(YarnApplicationState.class); return TypeConverter.fromYarnApps( client.getApplications(appTypes, appStates), this.conf); } catch (YarnException e) { throw new IOException(e); } }
public static org.apache.hadoop.mapreduce.JobID fromYarn(ApplicationId appID) { String identifier = fromClusterTimeStamp(appID.getClusterTimestamp()); return new org.apache.hadoop.mapred.JobID(identifier, appID.getId()); }
public static JobId toYarn(org.apache.hadoop.mapreduce.JobID id) { JobId jobId = recordFactory.newRecordInstance(JobId.class); jobId.setId(id.getId()); //currently there is 1-1 mapping between appid and jobid ApplicationId appId = ApplicationId.newInstance( toClusterTimeStamp(id.getJtIdentifier()), id.getId()); jobId.setAppId(appId); return jobId; }
public JobHistoryCopyService(ApplicationAttemptId applicationAttemptId, EventHandler handler) { super("JobHistoryCopyService"); this.applicationAttemptId = applicationAttemptId; this.jobId = TypeConverter.toYarn( TypeConverter.fromYarn(applicationAttemptId.getApplicationId())); this.handler = handler; }
public static QueueInfo fromYarn(org.apache.hadoop.yarn.api.records.QueueInfo queueInfo, Configuration conf) { QueueInfo toReturn = new QueueInfo(queueInfo.getQueueName(), "Capacity: " + queueInfo.getCapacity() * 100 + ", MaximumCapacity: " + (queueInfo.getMaximumCapacity() < 0 ? "UNDEFINED" : queueInfo.getMaximumCapacity() * 100) + ", CurrentCapacity: " + queueInfo.getCurrentCapacity() * 100, fromYarn(queueInfo.getQueueState()), TypeConverter.fromYarnApps(queueInfo.getApplications(), conf)); List<QueueInfo> childQueues = new ArrayList<QueueInfo>(); for(org.apache.hadoop.yarn.api.records.QueueInfo childQueue : queueInfo.getChildQueues()) { childQueues.add(fromYarn(childQueue, conf)); } toReturn.setQueueChildren(childQueues); return toReturn; }
public JobStatus[] getAllJobs() throws IOException, InterruptedException { try { Set<String> appTypes = new HashSet<String>(1); appTypes.add(MRJobConfig.MR_APPLICATION_TYPE); EnumSet<YarnApplicationState> appStates = EnumSet.noneOf(YarnApplicationState.class); return TypeConverter.fromYarnApps( client.getApplications(appTypes, appStates), this.conf); } catch (YarnException e) { throw new IOException(e); } }
public static org.apache.hadoop.mapreduce.JobID fromYarn(ApplicationId appID) { String identifier = fromClusterTimeStamp(appID.getClusterTimestamp()); return new org.apache.hadoop.mapred.JobID(identifier, appID.getId()); }
public static JobId toYarn(org.apache.hadoop.mapreduce.JobID id) { JobId jobId = recordFactory.newRecordInstance(JobId.class); jobId.setId(id.getId()); //currently there is 1-1 mapping between appid and jobid ApplicationId appId = ApplicationId.newInstance( toClusterTimeStamp(id.getJtIdentifier()), id.getId()); jobId.setAppId(appId); return jobId; }
public JobHistoryCopyService(ApplicationAttemptId applicationAttemptId, EventHandler<Event> handler) { super("JobHistoryCopyService"); this.applicationAttemptId = applicationAttemptId; this.jobId = TypeConverter.toYarn( TypeConverter.fromYarn(applicationAttemptId.getApplicationId())); this.handler = handler; }
public static String getApplicationWebURLOnJHSWithoutScheme(Configuration conf, ApplicationId appId) throws UnknownHostException { //construct the history url for job String addr = getJHSWebappURLWithoutScheme(conf); Iterator<String> it = ADDR_SPLITTER.split(addr).iterator(); it.next(); // ignore the bind host String port = it.next(); // Use hs address to figure out the host for webapp addr = conf.get(XLearningConfiguration.XLEARNING_HISTORY_ADDRESS, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_ADDRESS); String host = ADDR_SPLITTER.split(addr).iterator().next(); String hsAddress = JOINER.join(host, ":", port); InetSocketAddress address = NetUtils.createSocketAddr( hsAddress, getDefaultJHSWebappPort(), getDefaultJHSWebappURLWithoutScheme()); StringBuffer sb = new StringBuffer(); if (address.getAddress().isAnyLocalAddress() || address.getAddress().isLoopbackAddress()) { sb.append(InetAddress.getLocalHost().getCanonicalHostName()); } else { sb.append(address.getHostName()); } sb.append(":").append(address.getPort()); sb.append("/jobhistory/job/"); JobID jobId = TypeConverter.fromYarn(appId); sb.append(jobId.toString()); return sb.toString(); }
private void convertRawCountersToCounters() { if (this.counters == null && this.rawCounters != null) { this.counters = TypeConverter.toYarn(rawCounters); this.rawCounters = null; } }
public static QueueInfo fromYarn(org.apache.hadoop.yarn.api.records.QueueInfo queueInfo, Configuration conf) { QueueInfo toReturn = new QueueInfo(queueInfo.getQueueName(), "Capacity: " + queueInfo.getCapacity() * 100 + ", MaximumCapacity: " + (queueInfo.getMaximumCapacity() < 0 ? "UNDEFINED" : queueInfo.getMaximumCapacity() * 100) + ", CurrentCapacity: " + queueInfo.getCurrentCapacity() * 100, fromYarn(queueInfo.getQueueState()), TypeConverter.fromYarnApps(queueInfo.getApplications(), conf)); List<QueueInfo> childQueues = new ArrayList<QueueInfo>(); for(org.apache.hadoop.yarn.api.records.QueueInfo childQueue : queueInfo.getChildQueues()) { childQueues.add(fromYarn(childQueue, conf)); } toReturn.setQueueChildren(childQueues); return toReturn; }
public JobStatus[] getAllJobs() throws IOException, InterruptedException { try { Set<String> appTypes = new HashSet<String>(1); appTypes.add(TezConstants.TEZ_APPLICATION_TYPE); return TypeConverter.fromYarnApps(client.getApplications(appTypes), this.conf); } catch (YarnException e) { throw new IOException(e); } }
public static org.apache.hadoop.mapreduce.JobID fromYarn(ApplicationId appID) { String identifier = fromClusterTimeStamp(appID.getClusterTimestamp()); return new org.apache.hadoop.mapred.JobID(identifier, appID.getId()); }
public static JobId toYarn(org.apache.hadoop.mapreduce.JobID id) { JobId jobId = recordFactory.newRecordInstance(JobId.class); jobId.setId(id.getId()); //currently there is 1-1 mapping between appid and jobid ApplicationId appId = ApplicationId.newInstance( toClusterTimeStamp(id.getJtIdentifier()), id.getId()); jobId.setAppId(appId); return jobId; }
@Override public Map<JobId, Job> getAllJobs(ApplicationId appID) { if (LOG.isDebugEnabled()) { LOG.debug("Called getAllJobs(AppId): " + appID); } // currently there is 1 to 1 mapping between app and job id org.apache.hadoop.mapreduce.JobID oldJobID = TypeConverter.fromYarn(appID); Map<JobId, Job> jobs = new HashMap<JobId, Job>(); JobId jobID = TypeConverter.toYarn(oldJobID); jobs.put(jobID, getJob(jobID)); return jobs; }