/** {@inheritDoc} */ @Override public Config getConfig() { return _instanceConf.getConfig(); }
private MetricContext constructMetricContext(Configurable sysConfig, Logger log) { gobblin.configuration.State tmpState = new gobblin.configuration.State(sysConfig.getConfigAsProperties()); return GobblinMetrics.isEnabled(sysConfig.getConfig()) ? Instrumented.getMetricContext(tmpState, getClass()) : null; }
public FSSpecStore(GobblinInstanceEnvironment env, SpecSerDe specSerDe) throws IOException { this(env.getSysConfig().getConfig(), specSerDe, Optional.<Logger>absent()); }
public FSSpecStore(GobblinInstanceEnvironment env, SpecSerDe specSerDe, Optional<Logger> log) throws IOException { this(env.getSysConfig().getConfig(), specSerDe, log); }
@Override public JobSpecMonitor forJobCatalog(GobblinInstanceDriver instanceDriver, MutableJobCatalog jobCatalog) throws IOException { Config config = instanceDriver.getSysConfig().getConfig().getConfig(CONFIG_PREFIX).withFallback(DEFAULTS); return forConfig(config, jobCatalog); }
@Override public JobSpecMonitor forJobCatalog(GobblinInstanceDriver instanceDriver, MutableJobCatalog jobCatalog) throws IOException { Config config = instanceDriver.getSysConfig().getConfig().getConfig(CONFIG_PREFIX).withFallback(DEFAULTS); return forConfig(config, jobCatalog); }
@Override public GobblinInstancePlugin createPlugin(GobblinInstanceDriver instance) { Config sysConfig = instance.getSysConfig().getConfig(); if (!sysConfig.hasPath(PluginStaticKeys.LOGIN_USER_FULL_KEY)) { throw new RuntimeException("Missing required sys config: " + PluginStaticKeys.LOGIN_USER_FULL_KEY); } if (!sysConfig.hasPath(PluginStaticKeys.LOGIN_USER_KEYTAB_FILE_FULL_KEY)) { throw new RuntimeException("Missing required sys config: " + PluginStaticKeys.LOGIN_USER_KEYTAB_FILE_FULL_KEY); } String loginUser = sysConfig.getString(PluginStaticKeys.LOGIN_USER_FULL_KEY); String loginUserKeytabFile = sysConfig.getString(PluginStaticKeys.LOGIN_USER_KEYTAB_FILE_FULL_KEY); return new HadoopKerberosKeytabAuthenticationPlugin(instance, loginUser, loginUserKeytabFile); } }
public QuartzJobSpecScheduler(GobblinInstanceEnvironment env) { this(Optional.of(env.getLog()), env.getSysConfig().getConfig(), Optional.<SchedulerService>absent()); }
public boolean getDefaultInstrumentationEnabled() { return _gobblinEnv.isPresent() ? _gobblinEnv.get().isInstrumentationEnabled() : GobblinMetrics.isEnabled(getSysConfig().getConfig()); }
public boolean getDefaultInstrumentationEnabled() { return GobblinMetrics.isEnabled(getSysConfig().getConfig()); }
private boolean getDefaultInstrumentationEnabled() { return GobblinMetrics.isEnabled(getSysConfig().getConfig()); }
public ImmutableFSJobCatalog(GobblinInstanceEnvironment env) throws IOException { this(env.getSysConfig().getConfig(), null, Optional.of(env.getMetricContext()), env.isInstrumentationEnabled()); }
public DefaultGobblinInstanceDriverImpl(String instanceName, Configurable sysConfig, JobCatalog jobCatalog, JobSpecScheduler jobScheduler, JobExecutionLauncher jobLauncher, Optional<MetricContext> baseMetricContext, Optional<Logger> log, SharedResourcesBroker<GobblinScopeTypes> instanceBroker) { Preconditions.checkNotNull(jobCatalog); Preconditions.checkNotNull(jobScheduler); Preconditions.checkNotNull(jobLauncher); Preconditions.checkNotNull(sysConfig); _instanceName = instanceName; _log = log.or(LoggerFactory.getLogger(getClass())); _metricCtx = baseMetricContext.or(constructMetricContext(sysConfig, _log)); _instrumentationEnabled = null != _metricCtx && GobblinMetrics.isEnabled(sysConfig.getConfig()); _jobCatalog = jobCatalog; _jobScheduler = jobScheduler; _jobLauncher = jobLauncher; _sysConfig = sysConfig; _instanceCfg = ConfigAccessor.createFromGlobalConfig(_sysConfig.getConfig()); _callbacksDispatcher = new JobLifecycleListenersList(_jobCatalog, _jobScheduler, _log); _instanceBroker = instanceBroker; _metrics = new StandardMetrics(this); }
public ImmutableFSJobCatalog(GobblinInstanceEnvironment env, PathAlterationObserver observer) throws IOException { this(env.getSysConfig().getConfig(), observer, Optional.of(env.getMetricContext()), env.isInstrumentationEnabled()); }
/** * Returns the list of plugins as defined in the system configuration. These are the * defined in the PLUGINS_FULL_KEY config option. * The list also includes plugins that are automatically added by gobblin. * */ public List<GobblinInstancePluginFactory> getDefaultPlugins() { List<String> pluginNames = ConfigUtils.getStringList(getSysConfig().getConfig(), PLUGINS_FULL_KEY); List<GobblinInstancePluginFactory> pluginFactories = Lists.newArrayList(); // By default email notification plugin is added. if (!ConfigUtils.getBoolean(getSysConfig().getConfig(), EmailNotificationPlugin.EMAIL_NOTIFICATIONS_DISABLED_KEY, EmailNotificationPlugin.EMAIL_NOTIFICATIONS_DISABLED_DEFAULT)) { pluginFactories.add(new EmailNotificationPlugin.Factory()); } pluginFactories.addAll(Lists.transform(pluginNames, new Function<String, GobblinInstancePluginFactory>() { @Override public GobblinInstancePluginFactory apply(String input) { Class<? extends GobblinInstancePluginFactory> factoryClass; try { factoryClass = _aliasResolver.resolveClass(input); return factoryClass.newInstance(); } catch (ClassNotFoundException|InstantiationException|IllegalAccessException e) { throw new RuntimeException("Unable to instantiate plugin factory " + input + ": " + e, e); } } })); return pluginFactories; }
private HadoopKerberosKeytabAuthenticationPlugin(GobblinInstanceDriver instance, String loginUser, String loginUserKeytabFile) { super(instance); Config sysConfig = instance.getSysConfig().getConfig(); _loginUser = loginUser; _loginUserKeytabFile = loginUserKeytabFile; HadoopConfigLoader configLoader = new HadoopConfigLoader(sysConfig); _hadoopConf = configLoader.getConf(); }
public SharedResourcesBroker<GobblinScopeTypes> getDefaultInstanceBroker() { getLog().warn("Creating a default instance broker for job launcher. Objects may not be shared across all jobs in this instance."); SharedResourcesBrokerImpl<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(getSysConfig().getConfig(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); return globalBroker.newSubscopedBuilder(new SimpleScope<>(GobblinScopeTypes.INSTANCE, getInstanceName())).build(); }
public SharedResourcesBroker<GobblinScopeTypes> getDefaultInstanceBroker() { SharedResourcesBrokerImpl<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(getSysConfig().getConfig(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); return globalBroker.newSubscopedBuilder(new SimpleScope<>(GobblinScopeTypes.INSTANCE, getInstanceName())).build(); }
public SharedResourcesBroker<GobblinScopeTypes> getDefaultInstanceBroker() { SharedResourcesBrokerImpl<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(getSysConfig().getConfig(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); return globalBroker.newSubscopedBuilder(new SimpleScope<>(GobblinScopeTypes.INSTANCE, getInstanceName())).build(); }