/** * Constructs counter groups from job runtime statistics. Hive mangles Hadoop Counter data, * forming counter names with format "$groupName::$counterName". * * @param counterNameToValue mangled hadoop counters from hive. * @return counter groups by name. */ public static Map<String, CounterGroup> counterGroupInfoMap(Map<String, Double> counterNameToValue) { Counters counters = new Counters(); for (Map.Entry<String, ? extends Number> entry : counterNameToValue.entrySet()) { String key = entry.getKey(); Number value = entry.getValue(); String[] cNames = key.split("::"); String groupName = cNames[0]; String counterName = cNames[1]; Counter counter = counters.findCounter(groupName, counterName); counter.setValue(value.longValue()); } return CounterGroup.counterGroupsByName(counters); }
@JsonIgnore public void setJobStats(HadoopStepStats stats) { Counters counters = new Counters(); for (String groupName : stats.getCounterGroups()) { for (String counterName : stats.getCountersFor(groupName)) { Long counterValue = stats.getCounterValue(groupName, counterName); counters.findCounter(groupName, counterName).setValue(counterValue); } } setCounterGroupMap(CounterGroup.counterGroupsByName(counters)); } }
@JsonIgnore public void setJobStats(HadoopStepStats stats) { Counters counters = new Counters(); for (String groupName : stats.getCounterGroups()) { for (String counterName : stats.getCountersFor(groupName)) { Long counterValue = stats.getCounterValue(groupName, counterName); counters.findCounter(groupName, counterName).setValue(counterValue); } } setCounterGroupMap(CounterGroup.counterGroupsByName(counters)); } }
private Counters getEnumCounters(Enum[] keys) { Counters counters = new Counters(); for (Enum key : keys) { for (long i = 0; i < MAX_VALUE; ++i) { counters.incrCounter(key, i); } } return counters; }
private Counters getEnumCounters(String[] gNames, String[] cNames) { Counters counters = new Counters(); for (String gName : gNames) { for (String cName : cNames) { for (long i = 0; i < MAX_VALUE; ++i) { counters.incrCounter(gName, cName, i); } } } return counters; }
@SuppressWarnings("deprecation") @Test public void testCounterIteratorConcurrency() { Counters counters = new Counters(); counters.incrCounter("group1", "counter1", 1); Iterator<Group> iterator = counters.iterator(); counters.incrCounter("group2", "counter2", 1); iterator.next(); }
@Test public void testLegacyGetGroupNames() { Counters counters = new Counters(); // create 2 filesystem counter groups counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1); counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1); counters.incrCounter("group1", "counter1", 1); HashSet<String> groups = new HashSet<String>(counters.getGroupNames()); HashSet<String> expectedGroups = new HashSet<String>(); expectedGroups.add("group1"); expectedGroups.add("FileSystemCounters"); //Legacy Name expectedGroups.add("org.apache.hadoop.mapreduce.FileSystemCounter"); assertEquals(expectedGroups, groups); }
@SuppressWarnings("deprecation") @Test public void testGroupIteratorConcurrency() { Counters counters = new Counters(); counters.incrCounter("group1", "counter1", 1); Group group = counters.getGroup("group1"); Iterator<Counter> iterator = group.iterator(); counters.incrCounter("group1", "counter2", 1); iterator.next(); }
@Test public void testMakeCompactString() { final String GC1 = "group1.counter1:1"; final String GC2 = "group2.counter2:3"; Counters counters = new Counters(); counters.incrCounter("group1", "counter1", 1); assertEquals("group1.counter1:1", counters.makeCompactString()); counters.incrCounter("group2", "counter2", 3); String cs = counters.makeCompactString(); assertTrue("Bad compact string", cs.equals(GC1 + ',' + GC2) || cs.equals(GC2 + ',' + GC1)); }
@Test public void testFileSystemGroupIteratorConcurrency() { Counters counters = new Counters(); // create 2 filesystem counter groups counters.findCounter("fs1", FileSystemCounter.BYTES_READ).increment(1); counters.findCounter("fs2", FileSystemCounter.BYTES_READ).increment(1); // Iterate over the counters in this group while updating counters in // the group Group group = counters.getGroup(FileSystemCounter.class.getName()); Iterator<Counter> iterator = group.iterator(); counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1); assertTrue(iterator.hasNext()); iterator.next(); counters.findCounter("fs3", FileSystemCounter.BYTES_READ).increment(1); assertTrue(iterator.hasNext()); iterator.next(); }
@SuppressWarnings("deprecation") @Test public void testWriteWithLegacyNames() { Counters counters = new Counters(); counters.incrCounter(Task.Counter.MAP_INPUT_RECORDS, 1); counters.incrCounter(JobInProgress.Counter.DATA_LOCAL_MAPS, 1); counters.findCounter("FileSystemCounters", "FILE_BYTES_READ").increment(1); checkLegacyNames(counters); }
@SuppressWarnings("deprecation") @Test public void testReadWithLegacyNames() { Counters counters = new Counters(); counters.incrCounter(TaskCounter.MAP_INPUT_RECORDS, 1); counters.incrCounter(JobCounter.DATA_LOCAL_MAPS, 1); counters.findCounter("file", FileSystemCounter.BYTES_READ).increment(1); checkLegacyNames(counters); }
@Test(timeout = 5000) public void testTaskID() throws IOException, InterruptedException { JobID jobid = new JobID("1014873536921", 6); TaskID tid = new TaskID(jobid, TaskType.MAP, 0); org.apache.hadoop.mapred.TaskID tid1 = org.apache.hadoop.mapred.TaskID.downgrade(tid); org.apache.hadoop.mapred.TaskReport treport = new org.apache.hadoop.mapred.TaskReport(tid1, 0.0f, State.FAILED.toString(), null, TIPStatus.FAILED, 100, 100, new org.apache.hadoop.mapred.Counters()); Assert .assertEquals(treport.getTaskId(), "task_1014873536921_0006_m_000000"); Assert.assertEquals(treport.getTaskID().toString(), "task_1014873536921_0006_m_000000"); } }
@Test public void testCounterValue() { Counters counters = new Counters(); final int NUMBER_TESTS = 100; final int NUMBER_INC = 10;
public Counter getCounter(Enum<?> name) { try { Counters counters = new Counters(); counters.incrCounter(name, rep.getCounter(name).getValue()); return counters.findCounter(name); } catch (Exception e) { throw new RuntimeException(e); } } public Counter getCounter(String group, String name) {
public Counter getCounter(String group, String name) { try { Counters counters = new Counters(); counters.incrCounter(group, name, rep.getCounter(group, name).getValue()); return counters.findCounter(group, name); } catch (Exception e) { throw new RuntimeException(e); } } public void incrCounter(Enum<?> key, long amount) {
private void initializeHadoopCounter() { counters = new Counters(); Counters.Group fsGrp = counters.addGroup(FS_COUNTER_GROUP, FS_COUNTER_GROUP); fsGrp.addCounter(PigStatsUtil.HDFS_BYTES_READ, PigStatsUtil.HDFS_BYTES_READ, 0); fsGrp.addCounter(PigStatsUtil.HDFS_BYTES_WRITTEN, PigStatsUtil.HDFS_BYTES_WRITTEN, 0); }
private Counters covertToHadoopCounters(TezCounters tezCounters) { Counters counters = new Counters(); for (CounterGroup tezGrp : tezCounters) { Group grp = counters.addGroup(tezGrp.getName(), tezGrp.getDisplayName()); for (TezCounter counter : tezGrp) { grp.addCounter(counter.getName(), counter.getDisplayName(), counter.getValue()); } } return counters; }
@Test public void testZeroCounterGroupsToJson() throws SerializationException { Counters counters = new Counters(); String json = MapRedStatsLogModule.toJson(counters); assertEquals(json, "[]"); }