/** {@inheritDoc} */ @Override public Config getConfig() { return _instanceConf.getConfig(); }
private MetricContext constructMetricContext(Configurable sysConfig, Logger log) { gobblin.configuration.State tmpState = new gobblin.configuration.State(sysConfig.getConfigAsProperties()); return GobblinMetrics.isEnabled(sysConfig.getConfig()) ? Instrumented.getMetricContext(tmpState, getClass()) : null; }
/** {@inheritDoc} */ @Override public Properties getConfigAsProperties() { return _instanceConf.getConfigAsProperties(); }
private static JobLauncher createLauncher(Configurable _sysConfig, JobSpec _jobSpec, Logger _log, Optional<String> jobLauncherType, SharedResourcesBroker<GobblinScopeTypes> instanceBroker) { if (jobLauncherType.isPresent()) { return JobLauncherFactory.newJobLauncher(_sysConfig.getConfigAsProperties(), _jobSpec.getConfigAsProperties(), jobLauncherType.get(), instanceBroker); } else { _log.info("Creating auto jobLauncher for " + _jobSpec); try { return JobLauncherFactory.newJobLauncher(_sysConfig.getConfigAsProperties(), _jobSpec.getConfigAsProperties(), instanceBroker); } catch (Exception e) { throw new RuntimeException("JobLauncher creation failed: " + e, e); } } }
public FSSpecStore(GobblinInstanceEnvironment env, SpecSerDe specSerDe) throws IOException { this(env.getSysConfig().getConfig(), specSerDe, Optional.<Logger>absent()); }
public LocalJobLauncher(Configurable instanceConf, JobSpec jobSpec) throws Exception { this(JobConfigurationUtils.combineSysAndJobProperties(instanceConf.getConfigAsProperties(), jobSpec.getConfigAsProperties())); }
public FSSpecStore(GobblinInstanceEnvironment env, SpecSerDe specSerDe, Optional<Logger> log) throws IOException { this(env.getSysConfig().getConfig(), specSerDe, log); }
public MetricContext getDefaultMetricContext() { if (_gobblinEnv.isPresent()) { return _gobblinEnv.get().getMetricContext() .childBuilder(JobExecutionLauncher.class.getSimpleName()).build(); } gobblin.configuration.State fakeState = new gobblin.configuration.State(getSysConfig().getConfigAsProperties()); List<Tag<?>> tags = new ArrayList<>(); MetricContext res = Instrumented.getMetricContext(fakeState, Launcher.class, tags); return res; }
@Override public JobSpecMonitor forJobCatalog(GobblinInstanceDriver instanceDriver, MutableJobCatalog jobCatalog) throws IOException { Config config = instanceDriver.getSysConfig().getConfig().getConfig(CONFIG_PREFIX).withFallback(DEFAULTS); return forConfig(config, jobCatalog); }
public MetricContext getDefaultMetricContext() { gobblin.configuration.State fakeState = new gobblin.configuration.State(getSysConfig().getConfigAsProperties()); List<Tag<?>> tags = new ArrayList<>(); tags.add(new Tag<>(StandardMetrics.INSTANCE_NAME_TAG, getInstanceName())); MetricContext res = Instrumented.getMetricContext(fakeState, StandardGobblinInstanceDriver.class, tags); return res; }
@Override public JobSpecMonitor forJobCatalog(GobblinInstanceDriver instanceDriver, MutableJobCatalog jobCatalog) throws IOException { Config config = instanceDriver.getSysConfig().getConfig().getConfig(CONFIG_PREFIX).withFallback(DEFAULTS); return forConfig(config, jobCatalog); }
private MetricContext getDefaultMetricContext() { gobblin.configuration.State fakeState = new gobblin.configuration.State(getSysConfig().getConfigAsProperties()); return Instrumented.getMetricContext(fakeState, StandardGobblinInstanceLauncher.class); }
@Override public GobblinInstancePlugin createPlugin(GobblinInstanceDriver instance) { Config sysConfig = instance.getSysConfig().getConfig(); if (!sysConfig.hasPath(PluginStaticKeys.LOGIN_USER_FULL_KEY)) { throw new RuntimeException("Missing required sys config: " + PluginStaticKeys.LOGIN_USER_FULL_KEY); } if (!sysConfig.hasPath(PluginStaticKeys.LOGIN_USER_KEYTAB_FILE_FULL_KEY)) { throw new RuntimeException("Missing required sys config: " + PluginStaticKeys.LOGIN_USER_KEYTAB_FILE_FULL_KEY); } String loginUser = sysConfig.getString(PluginStaticKeys.LOGIN_USER_FULL_KEY); String loginUserKeytabFile = sysConfig.getString(PluginStaticKeys.LOGIN_USER_KEYTAB_FILE_FULL_KEY); return new HadoopKerberosKeytabAuthenticationPlugin(instance, loginUser, loginUserKeytabFile); } }
@Override protected void startUp() throws Exception { this.metrics.startMetricReporting(getInstance().getSysConfig().getConfigAsProperties()); }
public QuartzJobSpecScheduler(GobblinInstanceEnvironment env) { this(Optional.of(env.getLog()), env.getSysConfig().getConfig(), Optional.<SchedulerService>absent()); }
public boolean getDefaultInstrumentationEnabled() { return _gobblinEnv.isPresent() ? _gobblinEnv.get().isInstrumentationEnabled() : GobblinMetrics.isEnabled(getSysConfig().getConfig()); }
private boolean getDefaultInstrumentationEnabled() { return GobblinMetrics.isEnabled(getSysConfig().getConfig()); }
public boolean getDefaultInstrumentationEnabled() { return GobblinMetrics.isEnabled(getSysConfig().getConfig()); }
public ImmutableFSJobCatalog(GobblinInstanceEnvironment env) throws IOException { this(env.getSysConfig().getConfig(), null, Optional.of(env.getMetricContext()), env.isInstrumentationEnabled()); }
public DefaultGobblinInstanceDriverImpl(String instanceName, Configurable sysConfig, JobCatalog jobCatalog, JobSpecScheduler jobScheduler, JobExecutionLauncher jobLauncher, Optional<MetricContext> baseMetricContext, Optional<Logger> log, SharedResourcesBroker<GobblinScopeTypes> instanceBroker) { Preconditions.checkNotNull(jobCatalog); Preconditions.checkNotNull(jobScheduler); Preconditions.checkNotNull(jobLauncher); Preconditions.checkNotNull(sysConfig); _instanceName = instanceName; _log = log.or(LoggerFactory.getLogger(getClass())); _metricCtx = baseMetricContext.or(constructMetricContext(sysConfig, _log)); _instrumentationEnabled = null != _metricCtx && GobblinMetrics.isEnabled(sysConfig.getConfig()); _jobCatalog = jobCatalog; _jobScheduler = jobScheduler; _jobLauncher = jobLauncher; _sysConfig = sysConfig; _instanceCfg = ConfigAccessor.createFromGlobalConfig(_sysConfig.getConfig()); _callbacksDispatcher = new JobLifecycleListenersList(_jobCatalog, _jobScheduler, _log); _instanceBroker = instanceBroker; _metrics = new StandardMetrics(this); }