public void run(JobConf conf) throws Exception { _runningJob = new JobClient(conf).submitJob(conf); info("See " + _runningJob.getTrackingURL() + " for details."); _runningJob.waitForCompletion(); if(!_runningJob.isSuccessful()) { throw new Exception("Hadoop job:" + getId() + " failed!"); } // dump all counters Counters counters = _runningJob.getCounters(); for(String groupName: counters.getGroupNames()) { Counters.Group group = counters.getGroup(groupName); info("Group: " + group.getDisplayName()); for(Counter counter: group) info(counter.getDisplayName() + ":\t" + counter.getValue()); } }
@SuppressWarnings("deprecation") @Test public void testGroupIteratorConcurrency() { Counters counters = new Counters(); counters.incrCounter("group1", "counter1", 1); Group group = counters.getGroup("group1"); Iterator<Counter> iterator = group.iterator(); counters.incrCounter("group1", "counter2", 1); iterator.next(); }
@Test public void testFileSystemGroupIteratorConcurrency() { Counters counters = new Counters(); // create 2 filesystem counter groups counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1); counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1); // Iterate over the counters in this group while updating counters in // the group Group group = counters.getGroup(FileSystemCounter.class.getName()); Iterator<Counter> iterator = group.iterator(); counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1); assertTrue(iterator.hasNext()); iterator.next(); counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1); assertTrue(iterator.hasNext()); iterator.next(); }
/** * Find a counter given the group and the name. * @param group the name of the group * @param name the internal name of the counter * @return the counter for that name */ public synchronized Counter findCounter(String group, String name) { return getGroup(group).getCounterForName(name); }
/** * Increments the specified counter by the specified amount, creating it if * it didn't already exist. * @param group the name of the group * @param counter the internal name of the counter * @param amount amount by which counter is to be incremented */ public synchronized void incrCounter(String group, String counter, long amount) { getGroup(group).getCounterForName(counter).value += amount; }
@Override public String aggregateStats(String counterGrpName, String statType) { long value = 0; if (counters != null) { // In case of counters, aggregation is done by JobTracker / MR AM itself // so no need to aggregate, simply return the counter value for requested stat. value = counters.getGroup(counterGrpName).getCounter(statType); } return String.valueOf(value); }
/** * Find a counter given the group and the name. * @param group the name of the group * @param name the internal name of the counter * @return the counter for that name */ public synchronized Counter findCounter(String group, String name) { return getGroup(group).getCounterForName(name); }
/** * Find a counter by using strings * @param group the name of the group * @param id the id of the counter within the group (0 to N-1) * @param name the internal name of the counter * @return the counter for that name * @deprecated */ @Deprecated public synchronized Counter findCounter(String group, int id, String name) { return getGroup(group).getCounterForName(name); }
/** * Find a counter by using strings * @param group the name of the group * @param id the id of the counter within the group (0 to N-1) * @param name the internal name of the counter * @return the counter for that name * @deprecated */ @Deprecated public synchronized Counter findCounter(String group, int id, String name) { return getGroup(group).getCounterForName(name); }
/** * Increments the specified counter by the specified amount, creating it if * it didn't already exist. * @param group the name of the group * @param counter the internal name of the counter * @param amount amount by which counter is to be incremented */ public synchronized void incrCounter(String group, String counter, long amount) { getGroup(group).getCounterForName(counter).increment(amount); }
public synchronized Counter findCounter(String group, String name) { if (name.equals("MAP_INPUT_BYTES")) { LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " + "Use FileInputFormatCounters as group name and " + " BYTES_READ as counter name instead"); return findCounter(FileInputFormatCounter.BYTES_READ); } String newGroupKey = getNewGroupKey(group); if (newGroupKey != null) { group = newGroupKey; } return getGroup(group).getCounterForName(name); }
public synchronized Counter findCounter(String group, String name) { if (name.equals("MAP_INPUT_BYTES")) { LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " + "Use FileInputFormatCounters as group name and " + " BYTES_READ as counter name instead"); return findCounter(FileInputFormatCounter.BYTES_READ); } String newGroupKey = getNewGroupKey(group); if (newGroupKey != null) { group = newGroupKey; } return getGroup(group).getCounterForName(name); }
public synchronized Counter findCounter(String group, String name) { if (name.equals("MAP_INPUT_BYTES")) { LOG.warn("Counter name MAP_INPUT_BYTES is deprecated. " + "Use FileInputFormatCounters as group name and " + " BYTES_READ as counter name instead"); return findCounter(FileInputFormatCounter.BYTES_READ); } String newGroupKey = getNewGroupKey(group); if (newGroupKey != null) { group = newGroupKey; } return getGroup(group).getCounterForName(name); }
public static boolean isMainSuccessful(RunningJob runningJob) throws IOException { boolean succeeded = runningJob.isSuccessful(); if (succeeded) { Counters counters = runningJob.getCounters(); if (counters != null) { Counters.Group group = counters.getGroup(LauncherAMUtils.COUNTER_GROUP); if (group != null) { succeeded = group.getCounter(LauncherAMUtils.COUNTER_LAUNCHER_ERROR) == 0; } } } return succeeded; }
private static Long get(FlowStepStats step, String group, String value) throws IOException { if (step instanceof HadoopStepStats) { HadoopStepStats hadoopStep = (HadoopStepStats)step; org.apache.hadoop.mapred.Counters.Group counterGroup = hadoopStep.getRunningJob().getCounters().getGroup(group); if (counterGroup != null) { return counterGroup.getCounter(value); } LOG.info("Counter " + group + ":" + value + " not set."); return 0l; } else { return step.getCounterValue(group, value); } }
private List<MRTaskInfo> toMRTaskInfos(TaskReport[] taskReports) { List<MRTaskInfo> taskInfos = Lists.newArrayList(); for (TaskReport taskReport : taskReports) { taskInfos.add(new MRTaskInfo(taskReport.getTaskId(), taskReport.getState(), taskReport.getStartTime(), taskReport.getFinishTime(), taskReport.getProgress(), groupToMap(taskReport.getCounters().getGroup(TaskCounter.class.getName())))); } return taskInfos; }
private List<MRTaskInfo> toMRTaskInfos(TaskReport[] taskReports) { List<MRTaskInfo> taskInfos = Lists.newArrayList(); for (TaskReport taskReport : taskReports) { taskInfos.add(new MRTaskInfo(taskReport.getTaskId(), taskReport.getState(), taskReport.getStartTime(), taskReport.getFinishTime(), taskReport.getProgress(), groupToMap(taskReport.getCounters().getGroup(TaskCounter.class.getName())))); } return taskInfos; }
public static boolean hasCredentials(RunningJob runningJob) throws IOException { boolean output = false; Counters counters = runningJob.getCounters(); if (counters != null) { Counters.Group group = counters.getGroup(COUNTER_GROUP); if (group != null) { output = group.getCounter(COUNTER_OUTPUT_DATA) > 0; } } return output; } }
public static void verifyCounters(RunningJob runningJob, int expected) throws IOException { assertEquals(expected, runningJob.getCounters().getCounter(EnumCounter.MAP_RECORDS)); assertEquals(expected, runningJob.getCounters().getGroup("StringCounter") .getCounter("MapRecords")); }
public static void verifyCounters(RunningJob runningJob, int expected) throws IOException { assertEquals(expected, runningJob.getCounters().getCounter(EnumCounter.MAP_RECORDS)); assertEquals(expected, runningJob.getCounters().getGroup("StringCounter") .getCounter("MapRecords")); }