public MultiVersionCleanableDatasetBase(final FileSystem fs, final Properties props, Config config, Logger log) throws IOException { this(fs, props, Boolean.valueOf(props.getProperty(SIMULATE_KEY, SIMULATE_DEFAULT)), Boolean.valueOf(props.getProperty(SKIP_TRASH_KEY, SKIP_TRASH_DEFAULT)), Boolean.valueOf(props.getProperty(DELETE_EMPTY_DIRECTORIES_KEY, DELETE_EMPTY_DIRECTORIES_DEFAULT)), Boolean.valueOf(props.getProperty(DELETE_AS_OWNER_KEY, DELETE_AS_OWNER_DEFAULT)), ConfigUtils.getBoolean(config, IS_DATASET_BLACKLISTED_KEY, Boolean.valueOf(IS_DATASET_BLACKLISTED_DEFAULT)), log); }
public RetentionAction(Config actionConfig, FileSystem fs, Config jobConfig) { this.versionSelectionAliasResolver = new ClassAliasResolver<>(VersionSelectionPolicy.class); this.fs = fs; this.isSimulateMode = ConfigUtils.getBoolean(jobConfig, FsCleanableHelper.SIMULATE_KEY, Boolean.valueOf(FsCleanableHelper.SIMULATE_DEFAULT)); }
Optional<String> replacementPartitionDbName, Optional<String> replacementPartitionTableName) { boolean shouldReplacePartition = ConfigUtils.getBoolean(config, SHOULD_REPLACE_PARTITION_KEY, false);
@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { BasicDataSource basicDataSource = MysqlStateStore.newDataSource(config); String stateStoreTableName = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_DB_TABLE); boolean compressedValues = ConfigUtils.getBoolean(config, ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES); try { return new MysqlStateStore(basicDataSource, stateStoreTableName, compressedValues, stateClass); } catch (Exception e) { throw new RuntimeException("Failed to create MysqlStateStore with factory", e); } } }
private static Config addSystemConfigurationToConfig(Config config) { Map<String, String> confMap = Maps.newHashMap(); addBrokerKeys(confMap, System.getenv().entrySet()); addBrokerKeys(confMap, System.getProperties().entrySet()); Config systemConfig = ConfigFactory.parseMap(confMap); Config tmpConfig = config.withFallback(systemConfig); String brokerConfPath = DEFAULT_BROKER_CONF_FILE; if (tmpConfig.hasPath(BROKER_CONF_FILE_KEY)) { brokerConfPath = tmpConfig.getString(BROKER_CONF_FILE_KEY); } Config resourceConfig = ConfigFactory.parseResources(SharedResourcesBrokerFactory.class, brokerConfPath); config = config.withFallback(resourceConfig).withFallback(systemConfig); if (ConfigUtils.getBoolean(config, LOAD_HADOOP_CONFIGURATION, true)) { Map<String, String> hadoopConfMap = Maps.newHashMap(); Configuration hadoopConf = new Configuration(); hadoopConf.addResource("gobblin-site.xml"); addBrokerKeys(hadoopConfMap, hadoopConf); config = config.withFallback(ConfigFactory.parseMap(hadoopConfMap)); } return config; }
public GobblinClusterManager(String clusterName, String applicationId, Config config, Optional<Path> appWorkDirOptional) throws Exception { this.clusterName = clusterName; this.config = config; this.isStandaloneMode = ConfigUtils.getBoolean(config, GobblinClusterConfigurationKeys.STANDALONE_CLUSTER_MODE_KEY, GobblinClusterConfigurationKeys.DEFAULT_STANDALONE_CLUSTER_MODE); this.applicationId = applicationId; initializeHelixManager(); this.fs = buildFileSystem(config); this.appWorkDir = appWorkDirOptional.isPresent() ? appWorkDirOptional.get() : GobblinClusterUtils.getAppWorkDirPath(this.fs, clusterName, applicationId); initializeAppLauncherAndServices(); }
private ConversionConfig(Config config, Table table, String destinationFormat) { Preconditions.checkArgument(config.hasPath(DESTINATION_TABLE_KEY), String.format("Key %s.%s is not specified", destinationFormat, DESTINATION_TABLE_KEY)); Preconditions.checkArgument(config.hasPath(DESTINATION_DB_KEY), String.format("Key %s.%s is not specified", destinationFormat, DESTINATION_DB_KEY)); Preconditions.checkArgument(config.hasPath(DESTINATION_DATA_PATH_KEY), String.format("Key %s.%s is not specified", destinationFormat, DESTINATION_DATA_PATH_KEY)); // Required this.destinationFormat = destinationFormat; this.destinationTableName = resolveTemplate(config.getString(DESTINATION_TABLE_KEY), table); this.destinationStagingTableName = String.format("%s_%s", this.destinationTableName, "staging"); // Fixed and non-configurable this.destinationDbName = resolveTemplate(config.getString(DESTINATION_DB_KEY), table); this.destinationDataPath = resolveTemplate(config.getString(DESTINATION_DATA_PATH_KEY), table); // Optional this.destinationViewName = Optional.fromNullable(resolveTemplate(ConfigUtils.getString(config, DESTINATION_VIEW_KEY, null), table)); this.destinationTableProperties = convertKeyValueListToProperties(ConfigUtils.getStringList(config, DESTINATION_TABLE_PROPERTIES_LIST_KEY)); this.clusterBy = ConfigUtils.getStringList(config, CLUSTER_BY_KEY); this.numBuckets = Optional.fromNullable(ConfigUtils.getInt(config, NUM_BUCKETS_KEY, null)); this.hiveRuntimeProperties = convertKeyValueListToProperties(ConfigUtils.getStringList(config, HIVE_RUNTIME_PROPERTIES_LIST_KEY)); this.evolutionEnabled = ConfigUtils.getBoolean(config, EVOLUTION_ENABLED, false); this.updateViewAlwaysEnabled = ConfigUtils.getBoolean(config, UPDATE_VIEW_ALWAYS_ENABLED, true); this.rowLimit = Optional.fromNullable(ConfigUtils.getInt(config, ROW_LIMIT_KEY, null)); this.sourceDataPathIdentifier = ConfigUtils.getStringList(config, SOURCE_DATA_PATH_IDENTIFIER_KEY); }
/** * Returns the list of plugins as defined in the system configuration. These are the * defined in the PLUGINS_FULL_KEY config option. * The list also includes plugins that are automatically added by gobblin. * */ public List<GobblinInstancePluginFactory> getDefaultPlugins() { List<String> pluginNames = ConfigUtils.getStringList(getSysConfig().getConfig(), PLUGINS_FULL_KEY); List<GobblinInstancePluginFactory> pluginFactories = Lists.newArrayList(); // By default email notification plugin is added. if (!ConfigUtils.getBoolean(getSysConfig().getConfig(), EmailNotificationPlugin.EMAIL_NOTIFICATIONS_DISABLED_KEY, EmailNotificationPlugin.EMAIL_NOTIFICATIONS_DISABLED_DEFAULT)) { pluginFactories.add(new EmailNotificationPlugin.Factory()); } pluginFactories.addAll(Lists.transform(pluginNames, new Function<String, GobblinInstancePluginFactory>() { @Override public GobblinInstancePluginFactory apply(String input) { Class<? extends GobblinInstancePluginFactory> factoryClass; try { factoryClass = _aliasResolver.resolveClass(input); return factoryClass.newInstance(); } catch (ClassNotFoundException|InstantiationException|IllegalAccessException e) { throw new RuntimeException("Unable to instantiate plugin factory " + input + ": " + e, e); } } })); return pluginFactories; }
this.isTopologyCatalogEnabled = ConfigUtils.getBoolean(config, ServiceConfigKeys.GOBBLIN_SERVICE_TOPOLOGY_CATALOG_ENABLED_KEY, true); if (isTopologyCatalogEnabled) { this.isFlowCatalogEnabled = ConfigUtils.getBoolean(config, ServiceConfigKeys.GOBBLIN_SERVICE_FLOW_CATALOG_ENABLED_KEY, true); if (isFlowCatalogEnabled) { this.isSchedulerEnabled = ConfigUtils.getBoolean(config, ServiceConfigKeys.GOBBLIN_SERVICE_SCHEDULER_ENABLED_KEY, true); if (isSchedulerEnabled) { this.isRestLIServerEnabled = ConfigUtils.getBoolean(config, ServiceConfigKeys.GOBBLIN_SERVICE_RESTLI_SERVER_ENABLED_KEY, true); if (isRestLIServerEnabled) { this.isTopologySpecFactoryEnabled = ConfigUtils.getBoolean(config, ServiceConfigKeys.GOBBLIN_SERVICE_TOPOLOGY_SPEC_FACTORY_ENABLED_KEY, true); if (this.isTopologySpecFactoryEnabled) {
@Override protected HiveDataset computeNext() { while (this.tables.hasNext()) { DbAndTable dbAndTable = this.tables.next(); try (AutoReturnableObject<IMetaStoreClient> client = HiveDatasetFinder.this.clientPool.getClient()) { Table table = client.get().getTable(dbAndTable.getDb(), dbAndTable.getTable()); Config datasetConfig = getDatasetConfig(table); if (ConfigUtils.getBoolean(datasetConfig, HIVE_DATASET_IS_BLACKLISTED_KEY, DEFAULT_HIVE_DATASET_IS_BLACKLISTED_KEY)) { continue; } if (HiveDatasetFinder.this.eventSubmitter.isPresent()) { SlaEventSubmitter.builder().datasetUrn(dbAndTable.toString()) .eventSubmitter(HiveDatasetFinder.this.eventSubmitter.get()).eventName(DATASET_FOUND).build().submit(); } return createHiveDataset(table, datasetConfig); } catch (Throwable t) { log.error(String.format("Failed to create HiveDataset for table %s.%s", dbAndTable.getDb(), dbAndTable.getTable()), t); if (HiveDatasetFinder.this.eventSubmitter.isPresent()) { SlaEventSubmitter.builder().datasetUrn(dbAndTable.toString()) .eventSubmitter(HiveDatasetFinder.this.eventSubmitter.get()).eventName(DATASET_ERROR) .additionalMetadata(FAILURE_CONTEXT, t.toString()).build().submit(); } } } return endOfData(); } };
.getBoolean(config, EMAIL_NOTIFICATION_ON_SHUTDOWN_KEY, DEFAULT_EMAIL_NOTIFICATION_ON_SHUTDOWN);
ConfigUtils.getBoolean(state.getJobSpec().getConfig(), ConfigurationKeys.ALERT_EMAIL_ENABLED_KEY, false); boolean notificationEmailEnabled = ConfigUtils.getBoolean(state.getJobSpec().getConfig(), ConfigurationKeys.NOTIFICATION_EMAIL_ENABLED_KEY, false);