@Override public Timer.Context apply(@Nonnull MetricContext input) { return input.timer(name).time(); } });
@Test public void testContextAwareTimer() { ContextAwareTimer jobTotalDuration = this.context.contextAwareTimer(TOTAL_DURATION); Assert.assertEquals(this.context.getTimers().get(jobTotalDuration.getName()), jobTotalDuration.getInnerMetric()); Assert.assertEquals(jobTotalDuration.getContext(), this.context); Assert.assertEquals(jobTotalDuration.getName(), TOTAL_DURATION); jobTotalDuration.update(50, TimeUnit.SECONDS); jobTotalDuration.update(100, TimeUnit.SECONDS); jobTotalDuration.update(150, TimeUnit.SECONDS); Assert.assertEquals(jobTotalDuration.getCount(), 3l); Assert.assertEquals(jobTotalDuration.getSnapshot().getMin(), TimeUnit.SECONDS.toNanos(50l)); Assert.assertEquals(jobTotalDuration.getSnapshot().getMax(), TimeUnit.SECONDS.toNanos(150l)); Assert.assertTrue(jobTotalDuration.time().stop() >= 0l); }
@Override public ContextAwareTimer newMetric(MetricContext context, String name) { return new ContextAwareTimer(context, name); }
@Override protected void onClose() { StringBuilder message = new StringBuilder(); message.append("========================").append("\n"); message.append("Statistics for FileSystem: ").append(getUri()).append("\n"); message.append("------------------------").append("\n"); message.append("method\tcalls\tmean time(ns)\t99 percentile(ns)").append("\n"); for (ContextAwareTimer timer : this.allTimers) { if (timer.getCount() > 0) { message.append(timer.getName()).append("\t").append(timer.getCount()).append("\t"). append(timer.getSnapshot().getMean()).append("\t").append(timer.getSnapshot().get99thPercentile()).append("\n"); } } message.append("------------------------").append("\n"); log.info(message.toString()); super.onClose(); }
@Test public void testContextAwareTimerFactory() { Assert.assertTrue(ContextAwareMetricFactory.DEFAULT_CONTEXT_AWARE_TIMER_FACTORY.isInstance(this.timer)); Assert.assertFalse(ContextAwareMetricFactory.DEFAULT_CONTEXT_AWARE_TIMER_FACTORY.isInstance(this.counter)); Assert.assertEquals(this.timer.getContext(), this.context); Assert.assertEquals(this.timer.getName(), TOTAL_DURATION); }
@Test(enabled = false) public void testListFiles() throws IOException, URISyntaxException { HDFSRoot hdfsRoot = new HDFSRoot("/tmp/ListFiles"); MetricsFileSystemInstrumentation fs = (MetricsFileSystemInstrumentation) FileSystem.get(new URI(instrumentedURI), new Configuration()); fs.listFiles(hdfsRoot.getRootPath(), true); Assert.assertEquals(fs.listFilesTimer.getCount(), 1); Assert.assertEquals(fs.listStatusTimer.getCount(), 0); hdfsRoot.cleanupRoot(); }
@Override public void update(long duration, TimeUnit unit) { super.update(duration, unit); if (this.parentTimer.isPresent()) { this.parentTimer.get().update(duration, unit); } }
@Test(enabled = false) public void testCreate6() throws IOException, URISyntaxException { HDFSRoot hdfsRoot = new HDFSRoot("/tmp/create"); MetricsFileSystemInstrumentation fs = (MetricsFileSystemInstrumentation) FileSystem.get(new URI(instrumentedURI), new Configuration()); Path newFile = new Path("/tmp/create/newFile"); FSDataOutputStream fstream = fs.create(newFile, true, 300, (short)1, 1048576, null); Assert.assertEquals(fs.createTimer.getCount(), 1); fstream.close(); hdfsRoot.cleanupRoot(); }
@Override protected void onClose() { StringBuilder message = new StringBuilder(); message.append("========================").append("\n"); message.append("Statistics for FileSystem: ").append(getUri()).append("\n"); message.append("------------------------").append("\n"); message.append("method\tcalls\tmean time(ns)\t99 percentile(ns)").append("\n"); for (ContextAwareTimer timer : this.allTimers) { if (timer.getCount() > 0) { message.append(timer.getName()).append("\t").append(timer.getCount()).append("\t"). append(timer.getSnapshot().getMean()).append("\t").append(timer.getSnapshot().get99thPercentile()).append("\n"); } } message.append("------------------------").append("\n"); log.info(message.toString()); super.onClose(); }
/** * End the previous stage and record the time spent in that stage. */ public void endStage() { if (this.currentStage != null) { long time = System.currentTimeMillis() - this.currentStageStart; this.timings.add(new Stage(this.currentStage, time)); if (reportAsMetrics && submitter.getMetricContext().isPresent()) { String timerName = submitter.getNamespace() + "." + name + "." + this.currentStage; submitter.getMetricContext().get().timer(timerName).update(time, TimeUnit.MILLISECONDS); } } this.currentStage = null; }
@Override public Collection<HiveSpec> call() throws Exception { try (Timer.Context context = metricContext.timer(HIVE_SPEC_COMPUTATION_TIMER).time()) { return policy.getHiveSpecs(new Path(path)); } } });
@Test(enabled = false) public void testCreate7() throws IOException, URISyntaxException { HDFSRoot hdfsRoot = new HDFSRoot("/tmp/create"); MetricsFileSystemInstrumentation fs = (MetricsFileSystemInstrumentation) FileSystem.get(new URI(instrumentedURI), new Configuration()); FsPermission permission = new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.READ); Path newFile = new Path("/tmp/create/newFile"); FSDataOutputStream fstream = fs.create(newFile, permission, true, 100, (short)2, 1048576, null); Assert.assertEquals(fs.createTimer.getCount(), 1); fstream.close(); hdfsRoot.cleanupRoot(); }
@Override public void update(long duration, TimeUnit unit) { super.update(duration, unit); if (this.parentTimer.isPresent()) { this.parentTimer.get().update(duration, unit); } }
protected MetricContext(String name, MetricContext parent, List<Tag<?>> tags, boolean isRoot) throws NameConflictException { Preconditions.checkArgument(!Strings.isNullOrEmpty(name)); this.closer = Closer.create(); try { this.innerMetricContext = this.closer.register(new InnerMetricContext(this, name, parent, tags)); } catch(ExecutionException ee) { throw Throwables.propagate(ee); } this.contextAwareMetricsSet = Sets.newConcurrentHashSet(); this.notificationTargets = Maps.newConcurrentMap(); this.executorServiceOptional = Optional.absent(); this.notificationTimer = new ContextAwareTimer(this, GOBBLIN_METRICS_NOTIFICATIONS_TIMER_NAME); register(this.notificationTimer); if (!isRoot) { RootMetricContext.get().addMetricContext(this); } }
/** * Add timer metrics to {@link DistributedFileSystem#delete(Path, boolean)} */ public boolean delete (Path f, boolean recursive) throws IOException { try (Closeable context = new TimerContextWithLog(deleteTimer.time(), "delete", f, recursive)) { return super.delete (f, recursive); } }
@Test(enabled = false) public void testCreate4() throws IOException, URISyntaxException { HDFSRoot hdfsRoot = new HDFSRoot("/tmp/create"); MetricsFileSystemInstrumentation fs = (MetricsFileSystemInstrumentation) FileSystem.get(new URI(instrumentedURI), new Configuration()); Path newFile = new Path("/tmp/create/newFile"); FSDataOutputStream fstream = fs.create(newFile, true, 300, null); Assert.assertEquals(fs.createTimer.getCount(), 1); fstream.close(); hdfsRoot.cleanupRoot(); }
/** * End the previous stage and record the time spent in that stage. */ public void endStage() { if (this.currentStage != null) { long time = System.currentTimeMillis() - this.currentStageStart; this.timings.add(new Stage(this.currentStage, time)); if (reportAsMetrics && submitter.getMetricContext().isPresent()) { String timerName = submitter.getNamespace() + "." + name + "." + this.currentStage; submitter.getMetricContext().get().timer(timerName).update(time, TimeUnit.MILLISECONDS); } } this.currentStage = null; }
@Override public ContextAwareTimer newMetric(ContextAwareMetricFactoryArgs args) { if (args instanceof ContextAwareMetricFactoryArgs.SlidingTimeWindowArgs) { ContextAwareMetricFactoryArgs.SlidingTimeWindowArgs windowArgs = (ContextAwareMetricFactoryArgs.SlidingTimeWindowArgs)args; return new ContextAwareTimer(windowArgs.getContext(), windowArgs.getName(), windowArgs.getWindowSize(), windowArgs.getUnit()); } throw new UnsupportedOperationException("Unknown factory arguments to create ContextAwareTimer"); }
/** * Add timer metrics to {@link FileSystem#listFiles(Path, boolean)} */ public RemoteIterator<LocatedFileStatus> listFiles(Path f, boolean recursive) throws FileNotFoundException, IOException { try (Closeable context = new TimerContextWithLog(this.listFilesTimer.time(), "listFiles", f, recursive)) { return super.listFiles(f, recursive); } }
@Test(enabled = false) public void testCreate5() throws IOException, URISyntaxException { HDFSRoot hdfsRoot = new HDFSRoot("/tmp/create"); MetricsFileSystemInstrumentation fs = (MetricsFileSystemInstrumentation) FileSystem.get(new URI(instrumentedURI), new Configuration()); Path newFile = new Path("/tmp/create/newFile"); FSDataOutputStream fstream = fs.create(newFile, true, 300, (short)1, 1048576); Assert.assertEquals(fs.createTimer.getCount(), 1); fstream.close(); hdfsRoot.cleanupRoot(); }