/** * Called when the metric command is changed by "delta" units at time "time" * via user "user" * * @param time the time of the event * @param command the metric that is updated, e.g., the operation name * @param user the user that updated the metric * @param delta the amount of change in the metric, e.g., +1 */ public void recordMetric(long time, String command, String user, long delta) { RollingWindow window = getRollingWindow(command, user); window.incAt(time, delta); }
private String buildOpRecordName(TopWindow window) { return TOPMETRICS_METRICS_SOURCE_NAME + ".windowMs=" + window.getWindowLenMs(); }
/** * When an event occurs at the specified time, this method reflects that in * the rolling window. * <p/> * * @param time the time at which the event occurred * @param delta the delta that will be added to the window */ public void incAt(long time, long delta) { int bi = computeBucketIndex(time); Bucket bucket = buckets[bi]; // If the last time the bucket was updated is out of the scope of the // rolling window, reset the bucket. if (bucket.isStaleNow(time)) { bucket.safeReset(time); } bucket.inc(delta); }
TopWindow window = new TopWindow(windowLenMs); Set<String> metricNames = metricMap.keySet(); LOG.debug("iterating in reported metrics, size={} values={}", String metricName = entry.getKey(); RollingWindowMap rollingWindows = entry.getValue(); TopN topN = getTopUsersForMetric(time, metricName, rollingWindows); final int size = topN.size(); if (size == 0) { continue; Op op = new Op(metricName, topN.getTotal()); window.addOp(op); User user = new User(userEntry.getName(), userEntry.getValue()); op.addUser(user);
/** * Get the rolling window specified by metric and user. * * @param metric the updated metric * @param user the user that updated the metric * @return the rolling window */ private RollingWindow getRollingWindow(String metric, String user) { RollingWindowMap rwMap = metricMap.get(metric); if (rwMap == null) { rwMap = new RollingWindowMap(); RollingWindowMap prevRwMap = metricMap.putIfAbsent(metric, rwMap); if (prevRwMap != null) { rwMap = prevRwMap; } } RollingWindow window = rwMap.get(user); if (window != null) { return window; } window = new RollingWindow(windowLenMs, bucketsPerWindow); RollingWindow prevWindow = rwMap.putIfAbsent(user, window); if (prevWindow != null) { window = prevWindow; } return window; } }
/** * Flatten out the top window metrics into * {@link org.apache.hadoop.metrics2.MetricsRecord}s for consumption by * external metrics systems. Each metrics record added corresponds to the * reporting period a.k.a window length of the configured rolling windows. */ @Override public void getMetrics(MetricsCollector collector, boolean all) { if (!isMetricsSourceEnabled) { return; } for (final TopWindow window : getTopWindows()) { MetricsRecordBuilder rb = collector.addRecord(buildOpRecordName(window)) .setContext("dfs"); for (final Op op: window.getOps()) { rb.addCounter(buildOpTotalCountMetricsInfo(op), op.getTotalCount()); for (User user : op.getTopUsers()) { rb.addCounter(buildOpRecordMetricsInfo(op, user), user.getCount()); } } } }
TopN topN = new TopN(topUsersCnt); Iterator<Map.Entry<String, RollingWindow>> iterator = rollingWindows.entrySet().iterator(); while (iterator.hasNext()) { Map.Entry<String, RollingWindow> entry = iterator.next(); String userName = entry.getKey(); RollingWindow aWindow = entry.getValue(); long windowSum = aWindow.getSum(time);
public void report(long currTime, String userName, String cmd) { LOG.debug("a metric is reported: cmd: {} user: {}", cmd, userName); userName = UserGroupInformation.trimLoginMethod(userName); for (RollingWindowManager rollingWindowManager : rollingWindowManagers .values()) { rollingWindowManager.recordMetric(currTime, cmd, userName, 1); rollingWindowManager.recordMetric(currTime, TopConf.ALL_CMDS, userName, 1); } }
public TopMetrics(Configuration conf, int[] reportingPeriods) { logConf(conf); for (int i = 0; i < reportingPeriods.length; i++) { rollingWindowManagers.put(reportingPeriods[i], new RollingWindowManager( conf, reportingPeriods[i])); } isMetricsSourceEnabled = conf.getBoolean(DFSConfigKeys.NNTOP_ENABLED_KEY, DFSConfigKeys.NNTOP_ENABLED_DEFAULT); }
/** * Get a list of the current TopWindow statistics, one TopWindow per tracked * time interval. */ public List<TopWindow> getTopWindows() { long monoTime = Time.monotonicNow(); List<TopWindow> windows = Lists.newArrayListWithCapacity (rollingWindowManagers.size()); for (Entry<Integer, RollingWindowManager> entry : rollingWindowManagers .entrySet()) { TopWindow window = entry.getValue().snapshot(monoTime); windows.add(window); } return windows; }
/** * Safely reset the bucket state considering concurrent updates (inc) and * resets. * * @param time the current time */ void safeReset(long time) { // At any point in time, only one thread is allowed to reset the // bucket synchronized (this) { if (isStaleNow(time)) { // reset the value before setting the time, it allows other // threads to safely assume that the value is updated if the // time is not stale value.set(0); updateTime.set(time); } // else a concurrent thread has already reset it: do nothing } }
/** * @param windowLenMs The period that is covered by the window. This period must * be more than the buffering delays. * @param numBuckets number of buckets in the window */ RollingWindow(int windowLenMs, int numBuckets) { buckets = new Bucket[numBuckets]; for (int i = 0; i < numBuckets; i++) { buckets[i] = new Bucket(); } this.windowLenMs = windowLenMs; this.bucketSize = windowLenMs / numBuckets; if (this.bucketSize % bucketSize != 0) { throw new IllegalArgumentException( "The bucket size in the rolling window is not integer: windowLenMs= " + windowLenMs + " numBuckets= " + numBuckets); } }
private MetricsInfo buildOpTotalCountMetricsInfo(Op op) { return Interns.info("op=" + StringUtils.deleteWhitespace(op.getOpType()) + ".TotalCount", "Total operation count"); }
@Override public boolean equals(Object other) { if (other instanceof NameValuePair) { return compareTo((NameValuePair)other) == 0; } return false; }
/** * Called when the metric command is changed by "delta" units at time "time" * via user "user" * * @param time the time of the event * @param command the metric that is updated, e.g., the operation name * @param user the user that updated the metric * @param delta the amount of change in the metric, e.g., +1 */ public void recordMetric(long time, String command, String user, long delta) { RollingWindow window = getRollingWindow(command, user); window.incAt(time, delta); }
/** * Get value represented by this window at the specified time * <p/> * * If time lags behind the latest update time, the new updates are still * included in the sum * * @param time * @return number of events occurred in the past period */ public long getSum(long time) { long sum = 0; for (Bucket bucket : buckets) { boolean stale = bucket.isStaleNow(time); if (!stale) { sum += bucket.value.get(); } if (LOG.isDebugEnabled()) { long bucketTime = bucket.updateTime.get(); String timeStr = new Date(bucketTime).toString(); LOG.debug("Sum: + " + sum + " Bucket: updateTime: " + timeStr + " (" + bucketTime + ") isStale " + stale + " at " + time); } } return sum; }
private String buildOpRecordName(TopWindow window) { return TOPMETRICS_METRICS_SOURCE_NAME + ".windowMs=" + window.getWindowLenMs(); }
@Override public boolean equals(Object other) { if (other instanceof NameValuePair) { return compareTo((NameValuePair)other) == 0; } return false; }
/** * Called when the metric command is changed by "delta" units at time "time" * via user "user" * * @param time the time of the event * @param command the metric that is updated, e.g., the operation name * @param user the user that updated the metric * @param delta the amount of change in the metric, e.g., +1 */ public void recordMetric(long time, String command, String user, long delta) { RollingWindow window = getRollingWindow(command, user); window.incAt(time, delta); }