public ImmutableFSJobCatalog(Config sysConfig, PathAlterationObserver observer) throws IOException { this(sysConfig, observer, Optional.<MetricContext>absent(), GobblinMetrics.isEnabled(sysConfig)); }
private EventSubmitter buildEventSubmitter() { return new EventSubmitter.Builder(this.gobblinMetrics.get().getMetricContext(), GobblinYarnEventConstants.EVENT_NAMESPACE) .build(); }
@Override protected void startUp() throws Exception { GobblinMetrics.get(this.appId).startMetricReporting(this.properties); }
@Override protected void shutDown() throws Exception { GobblinMetrics.get(this.appId).stopMetricsReporting(); } }
@Test public void testCustomTags() { Properties testProperties = new Properties(); Tag<String> expectedPropertyTag = new Tag<>("key1", "value1"); GobblinMetrics.addCustomTagToProperties(testProperties, expectedPropertyTag); State testState = new State(testProperties); List<Tag<?>> tags = GobblinMetrics.getCustomTagsFromState(testState); Assert.assertEquals(Iterables.getFirst(tags, null), expectedPropertyTag); Tag<String> expectedStateTag = new Tag<>("key2", "value2"); GobblinMetrics.addCustomTagToState(testState, expectedStateTag); tags = GobblinMetrics.getCustomTagsFromState(testState); Assert.assertTrue(tags.containsAll(ImmutableList.of(expectedPropertyTag, expectedStateTag))); }
if (GobblinMetrics.isEnabled(this.taskState)) { GobblinMetrics forkMetrics = GobblinMetrics .get(getForkMetricsName(taskContext.getTaskMetrics(), this.taskState, index), taskContext.getTaskMetrics().getMetricContext(), getForkMetricsTags(this.taskState, index)); this.closer.register(forkMetrics.getMetricContext()); Instrumented.setMetricContextName(this.taskState, forkMetrics.getMetricContext().getName());
public MRCompactor(Properties props, List<? extends Tag<?>> tags, Optional<CompactorListener> compactorListener) throws IOException { this.state = new State(); this.state.addAll(props); this.initilizeTime = getCurrentTime(); this.tags = tags; this.conf = HadoopUtils.getConfFromState(this.state); this.tmpOutputDir = getTmpOutputDir(); this.fs = getFileSystem(); this.datasets = getDatasetsFinder().findDistinctDatasets(); this.jobExecutor = createJobExecutor(); this.jobRunnables = Maps.newConcurrentMap(); this.closer = Closer.create(); this.stopwatch = Stopwatch.createStarted(); this.gobblinMetrics = initializeMetrics(); this.eventSubmitter = new EventSubmitter.Builder( GobblinMetrics.get(this.state.getProp(ConfigurationKeys.JOB_NAME_KEY)).getMetricContext(), MRCompactor.COMPACTION_TRACKING_EVENTS_NAMESPACE).build(); this.compactorListener = compactorListener; this.dataVerifTimeoutMinutes = getDataVerifTimeoutMinutes(); this.compactionTimeoutMinutes = getCompactionTimeoutMinutes(); this.shouldVerifDataCompl = shouldVerifyDataCompleteness(); this.compactionCompleteListener = getCompactionCompleteListener(); this.verifier = this.shouldVerifDataCompl ? Optional.of(this.closer.register(new DataCompletenessVerifier(this.state))) : Optional.<DataCompletenessVerifier> absent(); this.shouldPublishDataIfCannotVerifyCompl = shouldPublishDataIfCannotVerifyCompl(); }
/** * Test the {@link GobblinMetrics} instance is removed from {@link GobblinMetricsRegistry} when * it stops metrics reporting */ public void testStopReportingMetrics() { String id = getClass().getSimpleName() + "-" + System.currentTimeMillis(); GobblinMetrics gobblinMetrics = GobblinMetrics.get(id); gobblinMetrics.startMetricReporting(new Properties()); Assert.assertEquals(GobblinMetricsRegistry.getInstance().get(id).get(), gobblinMetrics); gobblinMetrics.stopMetricsReporting(); Assert.assertFalse(GobblinMetricsRegistry.getInstance().get(id).isPresent()); } }
@Test public void testInstrumented() { GobblinMetrics gobblinMetrics = GobblinMetrics.get("parent.context"); State state = new State(); state.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(true)); state.setProp(Instrumented.METRIC_CONTEXT_NAME_KEY, gobblinMetrics.getName()); Instrumented instrumented = new Instrumented(state, InstrumentedExtractor.class); Assert.assertNotNull(instrumented.getMetricContext()); Assert.assertTrue(instrumented.getMetricContext().getParent().isPresent()); Assert.assertEquals(instrumented.getMetricContext().getParent().get(), gobblinMetrics.getMetricContext()); Map<String, ?> tags = instrumented.getMetricContext().getTagMap(); Map<String, String> expectedTags = new HashMap<>(); expectedTags.put("construct", Constructs.EXTRACTOR.toString()); expectedTags.put("class", InstrumentedExtractor.class.getCanonicalName()); expectedTags.put(MetricContext.METRIC_CONTEXT_ID_TAG_NAME, tags.get(MetricContext.METRIC_CONTEXT_ID_TAG_NAME).toString()); expectedTags.put(MetricContext.METRIC_CONTEXT_NAME_TAG_NAME, tags.get(MetricContext.METRIC_CONTEXT_NAME_TAG_NAME).toString()); Assert.assertEquals(tags.size(), expectedTags.size()); for (Map.Entry<String, ?> tag : tags.entrySet()) { Assert.assertTrue(expectedTags.containsKey(tag.getKey())); Assert.assertEquals(expectedTags.get(tag.getKey()), tag.getValue().toString()); } }
/** * Get a {@link GobblinMetrics} instance with the given ID. * * @param id the given {@link GobblinMetrics} ID * @return a {@link GobblinMetrics} instance */ public static GobblinMetrics get(String id) { return get(id, null); }
/** * Start metric reporting. * * @param configuration configuration properties */ public void startMetricReporting(Configuration configuration) { Properties props = new Properties(); for (Map.Entry<String, String> entry : configuration) { props.put(entry.getKey(), entry.getValue()); } startMetricReporting(props); }
/** * Add a {@link Tag} to a {@link Properties} with key {@link #METRICS_STATE_CUSTOM_TAGS}. * Also see {@link #addCustomTagToState(State, Tag)} * * <p> * The {@link Properties} passed can be used to build a {@link State}. * {@link org.apache.gobblin.metrics.Tag}s under this key can later be parsed using the method {@link #getCustomTagsFromState}. * </p> * * @param properties {@link Properties} to add the tag to. * @param tag {@link Tag} to add. */ public static void addCustomTagToProperties(Properties properties, Tag<?> tag) { // Build a state wrapper to add custom tag to property State state = new State(properties); addCustomTagToState(state, tag); }
GobblinMetrics.addCustomTagsToProperties(jobProps, clusterNameTags); GobblinMetrics.addCustomTagToState(this.jobContext.getJobState(), metadataTags);
@Override protected void shutDown() throws Exception { this.metrics.stopMetricsReporting(); super.shutDown(); } }
/** * Build a new {@link CopyDataPublisher} from {@link State}. The constructor expects the following to be set in the * {@link State}, * <ul> * <li>{@link ConfigurationKeys#WRITER_OUTPUT_DIR} * <li>{@link ConfigurationKeys#WRITER_FILE_SYSTEM_URI} * </ul> * */ public CopyDataPublisher(State state) throws IOException { super(state); // Extract LineageInfo from state if (state instanceof SourceState) { lineageInfo = LineageInfo.getLineageInfo(((SourceState) state).getBroker()); } else if (state instanceof WorkUnitState) { lineageInfo = LineageInfo.getLineageInfo(((WorkUnitState) state).getTaskBrokerNullable()); } else { lineageInfo = Optional.absent(); } String uri = this.state.getProp(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, ConfigurationKeys.LOCAL_FS_URI); this.fs = FileSystem.get(URI.create(uri), WriterUtils.getFsConfiguration(state)); FileAwareInputStreamDataWriterBuilder.setJobSpecificOutputPaths(state); this.writerOutputDir = new Path(state.getProp(ConfigurationKeys.WRITER_OUTPUT_DIR)); MetricContext metricContext = Instrumented.getMetricContext(state, CopyDataPublisher.class, GobblinMetrics.getCustomTagsFromState(state)); this.eventSubmitter = new EventSubmitter.Builder(metricContext, "org.apache.gobblin.copy.CopyDataPublisher").build(); this.recoveryHelper = new RecoveryHelper(this.fs, state); this.recoveryHelper.purgeOldPersistedFile(); }
@Override public void execute() throws IOException { LOG.info("Starting the clean-up steps."); try { serviceManager.stopAsync().awaitStopped(5, TimeUnit.SECONDS); } catch (TimeoutException te) { // Ignored } finally { if (jobMetrics.isPresent()) { try { jobMetrics.get().stopMetricsReporting(); } catch (Throwable throwable) { LOG.error("Failed to stop job metrics reporting.", throwable); } finally { GobblinMetrics.remove(jobMetrics.get().getName()); } } } } };
GobblinMetrics.addCustomTagsToProperties(jobProps, tags);
if (GobblinMetrics.isEnabled(this.taskState)) { GobblinMetrics forkMetrics = GobblinMetrics .get(getForkMetricsName(taskContext.getTaskMetrics(), this.taskState, index), taskContext.getTaskMetrics().getMetricContext(), getForkMetricsTags(this.taskState, index)); this.closer.register(forkMetrics.getMetricContext()); Instrumented.setMetricContextName(this.taskState, forkMetrics.getMetricContext().getName());
GobblinMetrics.get(this.dataset.jobProps().getProp(ConfigurationKeys.JOB_NAME_KEY)).getMetricContext(), MRCompactor.COMPACTION_TRACKING_EVENTS_NAMESPACE).build();
@Override protected void shutDown() throws Exception { GobblinMetrics.get(this.appId).stopMetricsReporting(); } }