/** * Return string value at <code>path</code> if <code>config</code> has path. If not return an empty string * * @param config in which the path may be present * @param path key to look for in the config object * @return string value at <code>path</code> if <code>config</code> has path. If not return an empty string */ public static String emptyIfNotPresent(Config config, String path) { return getString(config, path, StringUtils.EMPTY); }
public SelectAfterTimeBasedPolicy(Properties props) { this(ConfigUtils.propertiesToConfig(props)); }
/** * Convert a given {@link Config} instance to a {@link Properties} instance. * * @param config the given {@link Config} instance * @return a {@link Properties} instance */ public static Properties configToProperties(Config config) { return configToProperties(config, Optional.<String>absent()); }
private Kafka08ConsumerClient(Config config) { super(config); bufferSize = ConfigUtils.getInt(config, CONFIG_KAFKA_BUFFER_SIZE_BYTES, CONFIG_KAFKA_BUFFER_SIZE_BYTES_DEFAULT); clientName = ConfigUtils.getString(config, CONFIG_KAFKA_CLIENT_NAME, CONFIG_KAFKA_CLIENT_NAME_DEFAULT); fetchCorrelationId = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_REQUEST_CORRELATION_ID, CONFIG_KAFKA_FETCH_REQUEST_CORRELATION_ID_DEFAULT); fetchTopicRetries = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_TOPIC_NUM_TRIES, CONFIG_KAFKA_FETCH_TOPIC_NUM_TRIES_DEFAULT); fetchOffsetRetries = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_OFFSET_NUM_TRIES, CONFIG_KAFKA_FETCH_OFFSET_NUM_TRIES_DEFAULT); }
@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { BasicDataSource basicDataSource = MysqlStateStore.newDataSource(config); String stateStoreTableName = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_DB_TABLE); boolean compressedValues = ConfigUtils.getBoolean(config, ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES); try { return new MysqlStateStore(basicDataSource, stateStoreTableName, compressedValues, stateClass); } catch (Exception e) { throw new RuntimeException("Failed to create MysqlStateStore with factory", e); } } }
/** * Load job configuration from job configuration files stored in general file system, * located by Path * @param sysProps Gobblin framework configuration properties * @return a list of job configurations in the form of {@link java.util.Properties} */ public static List<Properties> loadGenericJobConfigs(Properties sysProps) throws ConfigurationException, IOException { Path rootPath = new Path(sysProps.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); PullFileLoader loader = new PullFileLoader(rootPath, rootPath.getFileSystem(new Configuration()), getJobConfigurationFileExtensions(sysProps), PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS); Config sysConfig = ConfigUtils.propertiesToConfig(sysProps); Collection<Config> configs = loader.loadPullFilesRecursively(rootPath, sysConfig, true); List<Properties> jobConfigs = Lists.newArrayList(); for (Config config : configs) { try { jobConfigs.add(resolveTemplate(ConfigUtils.configToProperties(config))); } catch (IOException ioe) { LOGGER.error("Could not parse job config at " + ConfigUtils.getString(config, ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, "Unknown path"), ioe); } } return jobConfigs; }
private ConversionConfig(Config config, Table table, String destinationFormat) { Preconditions.checkArgument(config.hasPath(DESTINATION_TABLE_KEY), String.format("Key %s.%s is not specified", destinationFormat, DESTINATION_TABLE_KEY)); Preconditions.checkArgument(config.hasPath(DESTINATION_DB_KEY), String.format("Key %s.%s is not specified", destinationFormat, DESTINATION_DB_KEY)); Preconditions.checkArgument(config.hasPath(DESTINATION_DATA_PATH_KEY), String.format("Key %s.%s is not specified", destinationFormat, DESTINATION_DATA_PATH_KEY)); // Required this.destinationFormat = destinationFormat; this.destinationTableName = resolveTemplate(config.getString(DESTINATION_TABLE_KEY), table); this.destinationStagingTableName = String.format("%s_%s", this.destinationTableName, "staging"); // Fixed and non-configurable this.destinationDbName = resolveTemplate(config.getString(DESTINATION_DB_KEY), table); this.destinationDataPath = resolveTemplate(config.getString(DESTINATION_DATA_PATH_KEY), table); // Optional this.destinationViewName = Optional.fromNullable(resolveTemplate(ConfigUtils.getString(config, DESTINATION_VIEW_KEY, null), table)); this.destinationTableProperties = convertKeyValueListToProperties(ConfigUtils.getStringList(config, DESTINATION_TABLE_PROPERTIES_LIST_KEY)); this.clusterBy = ConfigUtils.getStringList(config, CLUSTER_BY_KEY); this.numBuckets = Optional.fromNullable(ConfigUtils.getInt(config, NUM_BUCKETS_KEY, null)); this.hiveRuntimeProperties = convertKeyValueListToProperties(ConfigUtils.getStringList(config, HIVE_RUNTIME_PROPERTIES_LIST_KEY)); this.evolutionEnabled = ConfigUtils.getBoolean(config, EVOLUTION_ENABLED, false); this.updateViewAlwaysEnabled = ConfigUtils.getBoolean(config, UPDATE_VIEW_ALWAYS_ENABLED, true); this.rowLimit = Optional.fromNullable(ConfigUtils.getInt(config, ROW_LIMIT_KEY, null)); this.sourceDataPathIdentifier = ConfigUtils.getStringList(config, SOURCE_DATA_PATH_IDENTIFIER_KEY); }
this.clusterName = ConfigUtils.getString(config, CLUSTER_NAME_KEY, DEFAULT_CLUSTER_NAME); this.helixClusterName = ConfigUtils.getString(config, HELIX_CLUSTER_NAME_KEY, this.clusterName); this.nfsParentDir = appendSlash(ConfigUtils.getString(config, NFS_PARENT_DIR_KEY, DEFAULT_NFS_PARENT_DIR)); this.awsRegion = ConfigUtils.getString(config, AWS_REGION_KEY, DEFAULT_AWS_REGION); this.awsConfDir = appendSlash(ConfigUtils.getString(config, AWS_CONF_DIR, nfsParentDir + DEFAULT_AWS_CONF_DIR_POSTFIX)); this.masterAmiId = ConfigUtils.getString(config, MASTER_AMI_ID_KEY, DEFAULT_MASTER_AMI_ID); this.masterInstanceType = ConfigUtils.getString(config, MASTER_INSTANCE_TYPE_KEY, DEFAULT_MASTER_INSTANCE_TYPE); this.masterJvmMemory = ConfigUtils.getString(config, MASTER_JVM_MEMORY_KEY, DEFAULT_MASTER_JVM_MEMORY); this.workerAmiId = ConfigUtils.getString(config, WORKER_AMI_ID_KEY, DEFAULT_WORKER_AMI_ID); this.workerInstanceType = ConfigUtils.getString(config, WORKER_INSTANCE_TYPE_KEY, DEFAULT_WORKER_INSTANCE_TYPE); this.workerJvmMemory = ConfigUtils.getString(config, WORKER_JVM_MEMORY_KEY, DEFAULT_WORKER_JVM_MEMORY); this.minWorkers = ConfigUtils.getInt(config, MIN_WORKERS_KEY, DEFAULT_MIN_WORKERS); this.maxWorkers = ConfigUtils.getInt(config, MAX_WORKERS_KEY, DEFAULT_MAX_WORKERS); this.desiredWorkers = ConfigUtils.getInt(config, DESIRED_WORKERS_KEY, DEFAULT_DESIRED_WORKERS); ConfigUtils.getString(config, MASTER_JARS_KEY, nfsParentDir + DEFAULT_MASTER_JARS_POSTFIX)); this.masterS3ConfUri = appendSlash( ConfigUtils.getString(config, MASTER_S3_CONF_URI_KEY, DEFAULT_MASTER_S3_CONF_URI)); this.masterS3ConfFiles = ConfigUtils.getString(config, MASTER_S3_CONF_FILES_KEY, DEFAULT_MASTER_S3_CONF_FILES); this.masterS3JarsUri = ConfigUtils.getString(config, MASTER_S3_JARS_URI_KEY, DEFAULT_MASTER_S3_JARS_URI); this.masterS3JarsFiles = ConfigUtils.getString(config, MASTER_S3_JARS_FILES_KEY, DEFAULT_MASTER_S3_JARS_FILES); this.workerJarsDir = appendSlash(ConfigUtils.getString(config, WORKER_JARS_KEY, nfsParentDir + DEFAULT_WORKER_JARS_POSTFIX));
Integer.toString(ParallelRunner.DEFAULT_PARALLEL_RUNNER_THREADS))); jobConfig = ConfigUtils.propertiesToConfig(jobProps); this.eventBus, this.stateStores.taskStateStore, outputTaskStateDir); if (Task.getExecutionModel(ConfigUtils.configToState(jobConfig)).equals(ExecutionModel.STREAMING)) { ConfigUtils.getString(jobConfig, GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY, ""); final String rebalancerToReplace = "org.apache.helix.task.JobRebalancer"; final String rebalancerClassDesired = GobblinJobRebalancer.class.getName();
Properties properties = ConfigUtils.configToProperties(config); if (!properties.contains(ServiceBasedAppLauncher.APP_STOP_TIME_SECONDS)) { properties.setProperty(ServiceBasedAppLauncher.APP_STOP_TIME_SECONDS, Long.toString(300)); this.isTopologyCatalogEnabled = ConfigUtils.getBoolean(config, ServiceConfigKeys.GOBBLIN_SERVICE_TOPOLOGY_CATALOG_ENABLED_KEY, true); if (isTopologyCatalogEnabled) { this.isFlowCatalogEnabled = ConfigUtils.getBoolean(config, ServiceConfigKeys.GOBBLIN_SERVICE_FLOW_CATALOG_ENABLED_KEY, true); if (isFlowCatalogEnabled) { Optional<String> zkConnectionString = Optional.fromNullable(ConfigUtils.getString(config, ServiceConfigKeys.ZK_CONNECTION_STRING_KEY, null)); if (zkConnectionString.isPresent()) { this.isSchedulerEnabled = ConfigUtils.getBoolean(config, ServiceConfigKeys.GOBBLIN_SERVICE_SCHEDULER_ENABLED_KEY, true); if (isSchedulerEnabled) { this.isRestLIServerEnabled = ConfigUtils.getBoolean(config, ServiceConfigKeys.GOBBLIN_SERVICE_RESTLI_SERVER_ENABLED_KEY, true); if (isRestLIServerEnabled) { this.isTopologySpecFactoryEnabled = ConfigUtils.getBoolean(config, ServiceConfigKeys.GOBBLIN_SERVICE_TOPOLOGY_SPEC_FACTORY_ENABLED_KEY, true); if (this.isTopologySpecFactoryEnabled) {
/** * creates a new {@link BasicDataSource} * @param config the properties used for datasource instantiation * @return */ public static BasicDataSource newDataSource(Config config) { BasicDataSource basicDataSource = new BasicDataSource(); PasswordManager passwordManager = PasswordManager.getInstance(ConfigUtils.configToProperties(config)); basicDataSource.setDriverClassName(ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_DB_JDBC_DRIVER_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_DB_JDBC_DRIVER)); // MySQL server can timeout a connection so need to validate connections before use basicDataSource.setValidationQuery("select 1"); basicDataSource.setTestOnBorrow(true); basicDataSource.setDefaultAutoCommit(false); basicDataSource.setTimeBetweenEvictionRunsMillis(60000); basicDataSource.setUrl(config.getString(ConfigurationKeys.STATE_STORE_DB_URL_KEY)); basicDataSource.setUsername(passwordManager.readPassword( config.getString(ConfigurationKeys.STATE_STORE_DB_USER_KEY))); basicDataSource.setPassword(passwordManager.readPassword( config.getString(ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY))); basicDataSource.setMinEvictableIdleTimeMillis( ConfigUtils.getLong(config, ConfigurationKeys.STATE_STORE_DB_CONN_MIN_EVICTABLE_IDLE_TIME_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_DB_CONN_MIN_EVICTABLE_IDLE_TIME)); return basicDataSource; }
private void initWithSelectionPolicy(Config config, Properties jobProps) { String selectionPolicyKey = StringUtils.substringAfter(SELECTION_POLICY_CLASS_KEY, CONFIGURATION_KEY_PREFIX); String versionFinderKey = StringUtils.substringAfter(VERSION_FINDER_CLASS_KEY, CONFIGURATION_KEY_PREFIX); Preconditions.checkArgument( config.hasPath(versionFinderKey), String.format("Version finder class is required at %s in config %s", versionFinderKey, config.root().render(ConfigRenderOptions.concise()))); VersionFinderAndPolicyBuilder<T> builder = VersionFinderAndPolicy.builder(); builder.versionFinder(createVersionFinder(config.getString(versionFinderKey), config, jobProps)); if (config.hasPath(selectionPolicyKey)) { builder.versionSelectionPolicy(createSelectionPolicy( ConfigUtils.getString(config, selectionPolicyKey, SelectNothingPolicy.class.getName()), config, jobProps)); } for (Class<? extends RetentionActionFactory> factoryClass : RETENTION_ACTION_TYPES) { try { RetentionActionFactory factory = factoryClass.newInstance(); if (factory.canCreateWithConfig(config)) { builder.retentionAction((RetentionAction) factory.createRetentionAction(config, this.fs, ConfigUtils.propertiesToConfig(jobProps))); } } catch (InstantiationException | IllegalAccessException e) { Throwables.propagate(e); } } this.versionFindersAndPolicies.add(builder.build()); }
private static Properties resolveTemplate(Properties jobProps) throws IOException { try { if (jobProps.containsKey(ConfigurationKeys.JOB_TEMPLATE_PATH)) { Config jobConfig = ConfigUtils.propertiesToConfig(jobProps); Properties resolvedProps = ConfigUtils.configToProperties((ResourceBasedJobTemplate .forResourcePath(jobProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH), new PackagedTemplatesJobCatalogDecorator())) .getResolvedConfig(jobConfig)); return resolvedProps; } else { return jobProps; } } catch (JobTemplate.TemplateException | SpecNotFoundException | URISyntaxException exc) { throw new IOException(exc); } } }
public StreamingJobConfigurationManager(EventBus eventBus, Config config, MutableJobCatalog jobCatalog) { super(eventBus, config); this.stopTimeoutSeconds = ConfigUtils.getLong(config, GobblinClusterConfigurationKeys.STOP_TIMEOUT_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_STOP_TIMEOUT_SECONDS); this.fetchJobSpecExecutor = Executors.newSingleThreadExecutor( ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("FetchJobSpecExecutor"))); String specExecutorInstanceConsumerClassName = ConfigUtils.getString(config, GobblinClusterConfigurationKeys.SPEC_EXECUTOR_INSTANCE_CONSUMER_CLASS_KEY, GobblinClusterConfigurationKeys.DEFAULT_STREAMING_SPEC_EXECUTOR_INSTANCE_CONSUMER_CLASS); LOGGER.info("Using SpecExecutorInstanceConsumer ClassNameclass name/alias " + specExecutorInstanceConsumerClassName); try { ClassAliasResolver<SpecExecutorInstanceConsumer> aliasResolver = new ClassAliasResolver<>(SpecExecutorInstanceConsumer.class); this.specExecutorInstanceConsumer = (SpecExecutorInstanceConsumer) GobblinConstructorUtils.invokeFirstConstructor( Class.forName(aliasResolver.resolve(specExecutorInstanceConsumerClassName)), ImmutableList.<Object>of(config, jobCatalog), ImmutableList.<Object>of(config)); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new RuntimeException("Could not construct SpecExecutorInstanceConsumer " + specExecutorInstanceConsumerClassName, e); } }
@Override public DataWriter build() throws IOException { State state = this.destination.getProperties(); Properties taskProps = state.getProperties(); Config config = ConfigUtils.propertiesToConfig(taskProps); long commitTimeoutMillis = ConfigUtils.getLong(config, BatchedEventhubDataWriter.COMMIT_TIMEOUT_MILLIS_CONFIG, BatchedEventhubDataWriter.COMMIT_TIMEOUT_MILLIS_DEFAULT); long commitStepWaitTimeMillis = ConfigUtils.getLong(config, BatchedEventhubDataWriter.COMMIT_STEP_WAIT_TIME_CONFIG, BatchedEventhubDataWriter.COMMIT_STEP_WAIT_TIME_DEFAULT); double failureAllowance = ConfigUtils.getDouble(config, BatchedEventhubDataWriter.FAILURE_ALLOWANCE_PCT_CONFIG, BatchedEventhubDataWriter.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0; return AsyncWriterManager.builder() .config(config) .commitTimeoutMillis(commitTimeoutMillis) .commitStepWaitTimeInMillis(commitStepWaitTimeMillis) .failureAllowanceRatio(failureAllowance) .retriesEnabled(false) .asyncDataWriter(getAsyncDataWriter(taskProps)).maxOutstandingWrites(10000) .build(); } }
public KafkaJobMonitor(String topic, MutableJobCatalog catalog, Config config) { super(topic, ConfigUtils.getConfigOrEmpty(config, KAFKA_JOB_MONITOR_PREFIX), 1); this.jobCatalog = catalog; }
"Missing property " + ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR); this.config = ConfigUtils.propertiesToConfig(properties); this.prioritizer = Optional.absent(); this.maxToCopy = CopyResourcePool.fromConfig(ConfigUtils.getConfigOrEmpty(this.config, MAX_COPY_PREFIX));
public KafkaDataWriter(KafkaProducer producer, Config config) { super(ConfigUtils.configToState(config)); recordsProduced = getMetricContext().meter(KafkaWriterMetricNames.RECORDS_PRODUCED_METER); recordsWritten = getMetricContext().meter(KafkaWriterMetricNames.RECORDS_SUCCESS_METER); recordsFailed = getMetricContext().meter(KafkaWriterMetricNames.RECORDS_FAILED_METER); bytesWritten = new AtomicInteger(-1); this.topic = config.getString(KafkaWriterConfigurationKeys.KAFKA_TOPIC); this.commitTimeoutInNanos = ConfigUtils.getLong(config, KafkaWriterConfigurationKeys.COMMIT_TIMEOUT_MILLIS_CONFIG, KafkaWriterConfigurationKeys.COMMIT_TIMEOUT_MILLIS_DEFAULT) * MILLIS_TO_NANOS; this.commitStepWaitTimeMillis = ConfigUtils.getLong(config, KafkaWriterConfigurationKeys.COMMIT_STEP_WAIT_TIME_CONFIG, KafkaWriterConfigurationKeys.COMMIT_STEP_WAIT_TIME_DEFAULT); this.failureAllowance = ConfigUtils.getDouble(config, KafkaWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_CONFIG, KafkaWriterConfigurationKeys.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0; this.producer = producer; this.producerCallback = new Callback() { @Override public void onCompletion(RecordMetadata metadata, Exception exception) { if (null == exception) { recordsWritten.mark(); } else { log.debug("record failed to write", exception); recordsFailed.mark(); } } }; }
Config config; try { config = ConfigUtils.propertiesToConfig(taskState.getProperties()); } catch (Exception e) { LOG.warn("Failed to deserialize taskState into Config.. continuing with an empty config", e); long commitIntervalMillis = ConfigUtils.getLong(config, TaskConfigurationKeys.STREAMING_WATERMARK_COMMIT_INTERVAL_MILLIS, TaskConfigurationKeys.DEFAULT_STREAMING_WATERMARK_COMMIT_INTERVAL_MILLIS);
public MultiVersionCleanableDatasetBase(final FileSystem fs, final Properties props, Config config, Logger log) throws IOException { this(fs, props, Boolean.valueOf(props.getProperty(SIMULATE_KEY, SIMULATE_DEFAULT)), Boolean.valueOf(props.getProperty(SKIP_TRASH_KEY, SKIP_TRASH_DEFAULT)), Boolean.valueOf(props.getProperty(DELETE_EMPTY_DIRECTORIES_KEY, DELETE_EMPTY_DIRECTORIES_DEFAULT)), Boolean.valueOf(props.getProperty(DELETE_AS_OWNER_KEY, DELETE_AS_OWNER_DEFAULT)), ConfigUtils.getBoolean(config, IS_DATASET_BLACKLISTED_KEY, Boolean.valueOf(IS_DATASET_BLACKLISTED_DEFAULT)), log); }