public void run(JobConf conf) throws Exception { _runningJob = new JobClient(conf).submitJob(conf); info("See " + _runningJob.getTrackingURL() + " for details."); _runningJob.waitForCompletion(); if(!_runningJob.isSuccessful()) { throw new Exception("Hadoop job:" + getId() + " failed!"); } // dump all counters Counters counters = _runningJob.getCounters(); for(String groupName: counters.getGroupNames()) { Counters.Group group = counters.getGroup(groupName); info("Group: " + group.getDisplayName()); for(Counter counter: group) info(counter.getDisplayName() + ":\t" + counter.getValue()); } }
@Test public void testLegacyGetGroupNames() { Counters counters = new Counters(); // create 2 filesystem counter groups counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1); counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1); counters.incrCounter("group1", "counter1", 1); HashSet<String> groups = new HashSet<String>(counters.getGroupNames()); HashSet<String> expectedGroups = new HashSet<String>(); expectedGroups.add("group1"); expectedGroups.add("FileSystemCounters"); //Legacy Name expectedGroups.add("org.apache.hadoop.mapreduce.FileSystemCounter"); assertEquals(expectedGroups, groups); }
public Iterable<String> getGroupNames() { if (mapred != null) { return mapred.getGroupNames(); } else { return mapreduce.getGroupNames(); } }
@Deprecated private static List<Counter> safeGetStatsFromRunningJob(RunningJob job) { try { org.apache.hadoop.mapred.Counters allCounters = job.getCounters(); List<Counter> counters = new ArrayList<>(); for (String group : allCounters.getGroupNames()) { counters.addAll(getAllFromHadoopGroup(allCounters.getGroup(group))); } return counters; } catch (Exception e) { LOG.error("Error getting counters from job", e); return Collections.emptyList(); } }
@SuppressWarnings("unchecked") private static JSONObject toJSONFromCounters(Counters counters) { if (counters == null) { return null; } JSONObject groups = new JSONObject(); for (String gName : counters.getGroupNames()) { JSONObject group = new JSONObject(); for (Counter counter : counters.getGroup(gName)) { String cName = counter.getName(); Long cValue = counter.getValue(); group.put(cName, Long.toString(cValue)); } groups.put(gName, group); } return groups; } }
@SuppressWarnings("unchecked") private static JSONObject toJSONFromCounters(Counters counters) { if (counters == null) { return null; } JSONObject groups = new JSONObject(); for (String gName : counters.getGroupNames()) { JSONObject group = new JSONObject(); for (Counter counter : counters.getGroup(gName)) { String cName = counter.getName(); Long cValue = counter.getValue(); group.put(cName, Long.toString(cValue)); } groups.put(gName, group); } return groups; } }
@SuppressWarnings("unchecked") @Override public String toJSON() { if (counters == null) { return null; } JSONObject groups = new JSONObject(); groups.put(ACTION_TYPE_LABEL, getCurrentActionType().toString()); for (String gName : counters.getGroupNames()) { JSONObject group = new JSONObject(); for (Counter counter : counters.getGroup(gName)) { String cName = counter.getName(); Long cValue = counter.getValue(); group.put(cName, cValue); } groups.put(gName, group); } return groups.toJSONString(); } }
@SuppressWarnings("unchecked") @Override public String toJSON() { if (counters == null) { return null; } JSONObject groups = new JSONObject(); groups.put(ACTION_TYPE_LABEL, getCurrentActionType().toString()); for (String gName : counters.getGroupNames()) { JSONObject group = new JSONObject(); for (Counter counter : counters.getGroup(gName)) { String cName = counter.getName(); Long cValue = counter.getValue(); group.put(cName, cValue); } groups.put(gName, group); } return groups.toJSONString(); } }
counters = MRJobStats.getCounters(job); String groupName = getGroupName(counters.getGroupNames());
buff.append("\n------------------------------------------"+ "---------------------------------------------"); for (String groupName : totalCounters.getGroupNames()) { Group totalGroup = totalCounters.getGroup(groupName); Group mapGroup = mapCounters.getGroup(groupName);
for (String groupName : counters.getGroupNames()) { Counters.Group group = counters.getGroup(groupName); String displayGroupName = group.getDisplayName();
Counters totalCounters = job.getCounters(); for (String groupName : totalCounters.getGroupNames()) { Counters.Group totalGroup = totalCounters.getGroup(groupName); Counters.Group mapGroup = mapCounters.getGroup(groupName);