/** {@inheritDoc} */ @Override public void addCounter(Counter counter) { addCounter(counter.getName(), counter.getDisplayName(), 0); }
/** {@inheritDoc} */ @Override public void incrAllCounters(CounterGroupBase<Counter> rightGroup) { for (final Counter counter : rightGroup) cntrs.findCounter(name, counter.getName()).increment(counter.getValue()); }
/** * Create a {@link org.apache.gobblin.metrics.GobblinMetrics} instance for this job run from the Hadoop counters. */ @VisibleForTesting void countersToMetrics(GobblinMetrics metrics) throws IOException { Optional<Counters> counters = Optional.fromNullable(this.job.getCounters()); if (counters.isPresent()) { // Write job-level counters CounterGroup jobCounterGroup = counters.get().getGroup(MetricGroup.JOB.name()); for (Counter jobCounter : jobCounterGroup) { metrics.getCounter(jobCounter.getName()).inc(jobCounter.getValue()); } // Write task-level counters CounterGroup taskCounterGroup = counters.get().getGroup(MetricGroup.TASK.name()); for (Counter taskCounter : taskCounterGroup) { metrics.getCounter(taskCounter.getName()).inc(taskCounter.getValue()); } } }
/** {@inheritDoc} */ @Override public synchronized void incrAllCounters(AbstractCounters<Counter, CounterGroup> other) { for (CounterGroup group : other) { for (Counter counter : group) { findCounter(group.getName(), counter.getName()).increment(counter.getValue()); } } }
@Override protected void handleFailure(Counters counters) throws IOException { try (Connection conn = ConnectionFactory.createConnection(job.getConfiguration())) { TableName tableName = TableName.valueOf(COMMON_TABLE_NAME); CounterGroup g = counters.getGroup("undef"); Iterator<Counter> it = g.iterator(); while (it.hasNext()) { String keyString = it.next().getName(); byte[] key = Bytes.toBytes(keyString); HRegionLocation loc = conn.getRegionLocator(tableName).getRegionLocation(key, true); LOG.error("undefined row " + keyString + ", " + loc); } g = counters.getGroup("unref"); it = g.iterator(); while (it.hasNext()) { String keyString = it.next().getName(); byte[] key = Bytes.toBytes(keyString); HRegionLocation loc = conn.getRegionLocator(tableName).getRegionLocation(key, true); LOG.error("unreferred row " + keyString + ", " + loc); } } } }
protected void handleFailure(Counters counters) throws IOException { Configuration conf = job.getConfiguration(); TableName tableName = getTableName(conf); try (Connection conn = ConnectionFactory.createConnection(conf)) { try (RegionLocator rl = conn.getRegionLocator(tableName)) { CounterGroup g = counters.getGroup("undef"); Iterator<Counter> it = g.iterator(); while (it.hasNext()) { String keyString = it.next().getName(); byte[] key = Bytes.toBytes(keyString); HRegionLocation loc = rl.getRegionLocation(key, true); LOG.error("undefined row " + keyString + ", " + loc); } g = counters.getGroup("unref"); it = g.iterator(); while (it.hasNext()) { String keyString = it.next().getName(); byte[] key = Bytes.toBytes(keyString); HRegionLocation loc = rl.getRegionLocation(key, true); LOG.error("unreferred row " + keyString + ", " + loc); } } } } }
@Override protected Set<String> getCountersFor( Counters counters, String group ) { Set<String> results = new HashSet<>(); for( Counter counter : counters.getGroup( group ) ) results.add( counter.getName() ); return results; }
@Override public synchronized void addCounter(T counter) { counters.put(counter.getName(), counter); limits.incrCounters(); }
@Override public synchronized void addCounter(T counter) { counters.put(counter.getName(), counter); limits.incrCounters(); }
public CounterGroupInfo(String name, CounterGroup group, CounterGroup mg, CounterGroup rg) { this.counterGroupName = name; this.counter = new ArrayList<CounterInfo>(); for (Counter c : group) { Counter mc = mg == null ? null : mg.findCounter(c.getName()); Counter rc = rg == null ? null : rg.findCounter(c.getName()); CounterInfo cinfo = new CounterInfo(c, mc, rc); this.counter.add(cinfo); } }
public CounterGroupInfo(String name, CounterGroup group, CounterGroup mg, CounterGroup rg) { this.counterGroupName = name; this.counter = new ArrayList<CounterInfo>(); for (Counter c : group) { Counter mc = mg == null ? null : mg.findCounter(c.getName()); Counter rc = rg == null ? null : rg.findCounter(c.getName()); CounterInfo cinfo = new CounterInfo(c, mc, rc); this.counter.add(cinfo); } }
public TaskCounterGroupInfo(String name, CounterGroup group) { this.counterGroupName = name; this.counter = new ArrayList<TaskCounterInfo>(); for (Counter c : group) { TaskCounterInfo cinfo = new TaskCounterInfo(c.getName(), c.getValue()); this.counter.add(cinfo); } } }
private void setCounters( TaskReport taskReport ) { this.counters = new HashMap<>(); Counters hadoopCounters = taskReport.getTaskCounters(); for( CounterGroup group : hadoopCounters ) { Map<String, Long> values = new HashMap<String, Long>(); this.counters.put( group.getName(), values ); for( Counter counter : group ) values.put( counter.getName(), counter.getValue() ); } }
private void setCounters( TaskReport taskReport ) { this.counters = new HashMap<>(); Counters hadoopCounters = taskReport.getTaskCounters(); for( CounterGroup group : hadoopCounters ) { Map<String, Long> values = new HashMap<String, Long>(); this.counters.put( group.getName(), values ); for( Counter counter : group ) values.put( counter.getName(), counter.getValue() ); } }
@Override protected void reduce(Text key, Iterable<BytesWritable> values, Context context){ Counter cheeseCounter = context.getCounter(CounterEnum.CHEESE); Counter fleesCounter = context.getCounter(CounterEnum.FLEES); System.out.println(cheeseCounter.getName() + ": " + cheeseCounter.getValue()); System.out.println(fleesCounter.getName() + ": " + fleesCounter.getValue()); } }
public synchronized void readFields(DataInput in) throws IOException { displayName = Text.readString(in); counters.clear(); int size = WritableUtils.readVInt(in); for(int i=0; i < size; i++) { Counter counter = new Counter(); counter.readFields(in); counters.put(counter.getName(), counter); } }
public CounterInfo(Counter c, Counter mc, Counter rc) { this.name = c.getName(); this.totalCounterValue = c.getValue(); this.mapCounterValue = mc == null ? 0 : mc.getValue(); this.reduceCounterValue = rc == null ? 0 : rc.getValue(); } }
public CounterInfo(Counter c, Counter mc, Counter rc) { this.name = c.getName(); this.totalCounterValue = c.getValue(); this.mapCounterValue = mc == null ? 0 : mc.getValue(); this.reduceCounterValue = rc == null ? 0 : rc.getValue(); } }
@Override public synchronized void readFields(DataInput in) throws IOException { displayName = StringInterner.weakIntern(Text.readString(in)); counters.clear(); int size = WritableUtils.readVInt(in); for (int i = 0; i < size; i++) { T counter = newCounter(); counter.readFields(in); counters.put(counter.getName(), counter); limits.incrCounters(); } }
public synchronized void incrAllCounters(Counters other) { for (CounterGroup cg : other) { for (Counter c : cg) { findCounter(cg.getName(), c.getName()).increment(c.getValue()); } } } }