public ImmutableFSJobCatalog(Config sysConfig, PathAlterationObserver observer) throws IOException { this(sysConfig, observer, Optional.<MetricContext>absent(), GobblinMetrics.isEnabled(sysConfig)); }
public Instrumented(State state, Class<?> klazz, List<Tag<?>> tags) { this.closer = Closer.create(); this.instrumentationEnabled = GobblinMetrics.isEnabled(state); this.metricContext = this.closer.register(getMetricContext(state, klazz, tags)); }
protected InstrumentedRowLevelPolicyBase(State state, Type type, Optional<Class<?>> classTag) { super(state, type); this.instrumentationEnabled = GobblinMetrics.isEnabled(state); this.closer = Closer.create(); this.metricContext = this.closer.register(Instrumented.getMetricContext(state, classTag.or(this.getClass()))); regenerateMetrics(); }
protected void init(WorkUnitState workUnitState, Class<?> classTag) throws Exception { this.instrumentationEnabled = GobblinMetrics.isEnabled(workUnitState); this.metricContext = this.closer.register(Instrumented.getMetricContext(workUnitState, this.classTag.or(classTag))); regenerateMetrics(); }
protected InstrumentedExtractorBase(WorkUnitState workUnitState, Optional<Class<?>> classTag) { super(); this.closer = Closer.create(); this.instrumentationEnabled = GobblinMetrics.isEnabled(workUnitState); this.metricContext = this.closer.register( Instrumented.getMetricContext(workUnitState, classTag.or(this.getClass()), generateTags(workUnitState))); regenerateMetrics(); }
protected InstrumentedDataWriterBase(State state, Optional<Class<?>> classTag) { this.closer = Closer.create(); this.instrumentationEnabled = GobblinMetrics.isEnabled(state); this.metricContext = this.closer.register(Instrumented.getMetricContext(state, classTag.or(this.getClass()))); if (this.instrumentationEnabled) { this.writerMetricsUpdater = Optional.of(buildWriterMetricsUpdater()); scheduleWriterMetricsUpdater(this.writerMetricsUpdater.get(), getWriterMetricsUpdaterInterval(state)); } else { this.writerMetricsUpdater = Optional.absent(); } regenerateMetrics(); }
private void addMetricsService(Properties properties) { if (GobblinMetrics.isEnabled(properties)) { addService(new MetricsReportingService(properties, this.appId)); } }
private Optional<ContainerMetrics> buildContainerMetrics() { Properties properties = ConfigUtils.configToProperties(this.config); if (GobblinMetrics.isEnabled(properties)) { return Optional.of(ContainerMetrics .get(ConfigUtils.configToState(config), this.applicationName, this.taskRunnerId)); } else { return Optional.absent(); } }
public boolean getDefaultInstrumentationEnabled() { return _gobblinEnv.isPresent() ? _gobblinEnv.get().isInstrumentationEnabled() : GobblinMetrics.isEnabled(getSysConfig().getConfig()); }
private boolean getDefaultInstrumentationEnabled() { return GobblinMetrics.isEnabled(getSysConfig().getConfig()); }
protected Converter<SI, SO, DI, DO> init(WorkUnitState workUnit, Class<?> classTag) { Converter<SI, SO, DI, DO> converter = super.init(workUnit); this.instrumentationEnabled = GobblinMetrics.isEnabled(workUnit); this.metricContext = this.closer.register(Instrumented.getMetricContext(workUnit, classTag)); regenerateMetrics(); return converter; }
public boolean getDefaultInstrumentationEnabled() { return GobblinMetrics.isEnabled(getSysConfig().getConfig()); }
public ComplianceJob(Properties properties) { this.properties = properties; ExecutorService executor = ScalingThreadPoolExecutor.newScalingThreadPool(0, Integer.parseInt(properties.getProperty(ComplianceConfigurationKeys.MAX_CONCURRENT_DATASETS, ComplianceConfigurationKeys.DEFAULT_MAX_CONCURRENT_DATASETS)), 100, ExecutorsUtils.newThreadFactory(Optional.<Logger>absent(), Optional.of("complaince-job-pool-%d"))); this.service = MoreExecutors.listeningDecorator(executor); this.closer = Closer.create(); List<Tag<?>> tags = Lists.newArrayList(); tags.addAll(Tag.fromMap(AzkabanTags.getAzkabanTags())); this.metricContext = this.closer.register(Instrumented.getMetricContext(new State(properties), ComplianceJob.class, tags)); this.isMetricEnabled = GobblinMetrics.isEnabled(properties); this.eventSubmitter = new EventSubmitter.Builder(this.metricContext, ComplianceEvents.NAMESPACE).build(); this.throwables = Lists.newArrayList(); }
public ThrottleWriter(DataWriter<D> writer, State state) { Preconditions.checkNotNull(writer, "DataWriter is required."); Preconditions.checkNotNull(state, "State is required."); this.state = state; this.writer = writer; this.type = ThrottleType.valueOf(state.getProp(WRITER_THROTTLE_TYPE_KEY)); int rateLimit = computeRateLimit(state); LOG.info("Rate limit for each writer: " + rateLimit + " " + type); this.limiter = new RateBasedLimiter(computeRateLimit(state)); if (GobblinMetrics.isEnabled(state)) { throttledTimer = Optional.<Timer>of(Instrumented.getMetricContext(state, getClass()).timer(WRITES_THROTTLED_TIMER)); } else { throttledTimer = Optional.absent(); } }
private MetricContext constructMetricContext(Configurable sysConfig, Logger log) { org.apache.gobblin.configuration.State tmpState = new org.apache.gobblin.configuration.State(sysConfig.getConfigAsProperties()); return GobblinMetrics.isEnabled(sysConfig.getConfig()) ? Instrumented.getMetricContext(tmpState, getClass()) : null; }
@VisibleForTesting UnsampledReport createUnsampledReports(UnsampledReport request) throws IOException { long startTimeInMillis = System.currentTimeMillis(); try { UnsampledReport requestedReport = requestUnsampledReport(request); UnsampledReport createdReport = pollForCompletion(wuState, gaService, requestedReport); createdReport.setEndDate(requestedReport.getEndDate()); return createdReport; } finally { long delta = System.currentTimeMillis() - startTimeInMillis; if (GobblinMetrics.isEnabled(wuState)) { Timer timer = Instrumented.getMetricContext(wuState, getClass()).timer(GA_UNSAMPLED_REPORT_CREATION_TIMER); Instrumented.updateTimer(Optional.of(timer), delta, TimeUnit.MILLISECONDS); } } }
protected void updateTaskMetrics() { if (GobblinMetrics.isEnabled(this.task.getTaskState().getWorkunit())) { this.task.updateRecordMetrics(); this.task.updateByteMetrics(); } } }
@Override protected void updateTaskMetrics() { super.updateTaskMetrics(); WorkUnit workUnit = this.task.getTaskState().getWorkunit(); if (GobblinMetrics.isEnabled(workUnit)) { if (workUnit.getPropAsBoolean(ConfigurationKeys.MR_REPORT_METRICS_AS_COUNTERS_KEY, ConfigurationKeys.DEFAULT_MR_REPORT_METRICS_AS_COUNTERS)) { updateCounters(this.task); } } // Tell the TaskTracker it's making progress this.context.progress(); } }
@Override public void close() throws IOException { try { this.cancellationExecutor.shutdownNow(); try { this.jobContext.getSource().shutdown(this.jobContext.getJobState()); } finally { if (GobblinMetrics.isEnabled(this.jobProps)) { GobblinMetricsRegistry.getInstance().remove(this.jobContext.getJobId()); } } } finally { unlockJob(); } }
@Override public void onTaskCommitCompletion(Task task) { WorkUnit workUnit = task.getTaskState().getWorkunit(); if (GobblinMetrics.isEnabled(workUnit)) { task.updateRecordMetrics(); task.updateByteMetrics(); if (workUnit.getPropAsBoolean(ConfigurationKeys.MR_REPORT_METRICS_AS_COUNTERS_KEY, ConfigurationKeys.DEFAULT_MR_REPORT_METRICS_AS_COUNTERS)) { updateCounters(task); } } LOG.info(String .format("Task %s completed running in %dms with state %s", task.getTaskId(), task.getTaskState().getTaskDuration(), task.getTaskState().getWorkingState())); }