/** * Return string value at <code>path</code> if <code>config</code> has path. If not return an empty string * * @param config in which the path may be present * @param path key to look for in the config object * @return string value at <code>path</code> if <code>config</code> has path. If not return an empty string */ public static String emptyIfNotPresent(Config config, String path) { return getString(config, path, StringUtils.EMPTY); }
private static String getPattern(Config config) { return ConfigUtils.getString(config, FLAKY_ERROR_REGEX_PATTERN_KEY, null); }
@VisibleForTesting AccessControlAction(Config actionConfig, FileSystem fs, Config jobConfig) { super(actionConfig, fs, jobConfig); this.permission = actionConfig.hasPath(MODE_KEY) ? Optional.of(new FsPermission(actionConfig.getString(MODE_KEY))) : Optional .<FsPermission> absent(); this.owner = Optional.fromNullable(ConfigUtils.getString(actionConfig, OWNER_KEY, null)); this.group = Optional.fromNullable(ConfigUtils.getString(actionConfig, GROUP_KEY, null)); this.selectionPolicy = createSelectionPolicy(actionConfig, jobConfig); }
/** * Build the {@link HelixManager} for the Application Master. */ private HelixManager buildHelixManager(Config config, String zkConnectionString) { String helixInstanceName = ConfigUtils.getString(config, GobblinClusterConfigurationKeys.HELIX_INSTANCE_NAME_KEY, GobblinClusterManager.class.getSimpleName()); return HelixManagerFactory.getZKHelixManager( config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY), helixInstanceName, InstanceType.CONTROLLER, zkConnectionString); }
private static ErrorType getType(Config config) { String type = ConfigUtils.getString(config, ERROR_TYPE_CONFIGURATION_KEY, ""); ErrorType errorType; if (!type.isEmpty()) { errorType = ErrorType.valueOf(type.toUpperCase()); } else { errorType = DEFAULT_ERROR_TYPE; } return errorType; }
public DatePartitionHiveVersionFinder(FileSystem fs, Config config) { this.pattern = ConfigUtils.getString(config, PARTITION_VALUE_DATE_TIME_PATTERN_KEY, DEFAULT_PARTITION_VALUE_DATE_TIME_PATTERN); if (config.hasPath(PARTITION_VALUE_DATE_TIME_TIMEZONE_KEY)) { this.formatter = DateTimeFormat.forPattern(pattern) .withZone(DateTimeZone.forID(config.getString(PARTITION_VALUE_DATE_TIME_TIMEZONE_KEY))); } else { this.formatter = DateTimeFormat.forPattern(pattern).withZone(DateTimeZone.forID(DEFAULT_PARTITION_VALUE_DATE_TIME_TIMEZONE)); } this.partitionKeyName = ConfigUtils.getString(config, PARTITION_KEY_NAME_KEY, DEFAULT_PARTITION_KEY_NAME); this.partitionKeyNamePredicate = new Predicate<FieldSchema>() { @Override public boolean apply(FieldSchema input) { return StringUtils.equalsIgnoreCase(input.getName(), DatePartitionHiveVersionFinder.this.partitionKeyName); } }; }
@Override public DatasetAwareMetadataProvider createMetadataProvider(Config metaConfig) { String permission = ConfigUtils.getString(metaConfig, ALL_PERMISSOIN, ""); log.info("User defined permission is: " + permission); return new SimpleConfigMetadataProvider(permission); } }
public HiveDatasetVersionCleaner(DatasetVersion datasetVersion, CleanableDataset cleanableDataset) { super(datasetVersion, cleanableDataset); Preconditions.checkArgument(cleanableDataset instanceof CleanableHiveDataset, String.format("%s only supports %s, " + "found: %s", this.getClass(), CleanableHiveDataset.class, cleanableDataset.getClass())); Preconditions.checkArgument(datasetVersion instanceof HiveDatasetVersion, String.format("%s only supports %s, " + "found: %s", this.getClass(), HiveDatasetVersionCleaner.class, datasetVersion.getClass())); this.cleanableHiveDataset = (CleanableHiveDataset) cleanableDataset; this.hiveDatasetVersion = (HiveDatasetVersion) datasetVersion; // For post cleanup activity: // Get db / table name from which partition has to be replaced-in for the target partition being deleted. this.replacementDbName = Optional.fromNullable(ConfigUtils.getString(cleanableHiveDataset.getDatasetConfig(), REPLACEMENT_HIVE_DB_NAME_KEY, null)); this.replacementTableName = Optional.fromNullable(ConfigUtils.getString(cleanableHiveDataset.getDatasetConfig(), REPLACEMENT_HIVE_TABLE_NAME_KEY, null)); }
/** * Build the {@link HelixManager} for the Service Master. */ private HelixManager buildHelixManager(Config config, String zkConnectionString) { String helixClusterName = config.getString(ServiceConfigKeys.HELIX_CLUSTER_NAME_KEY); String helixInstanceName = ConfigUtils.getString(config, ServiceConfigKeys.HELIX_INSTANCE_NAME_KEY, GobblinServiceManager.class.getSimpleName()); LOGGER.info("Creating Helix cluster if not already present [overwrite = false]: " + zkConnectionString); HelixUtils.createGobblinHelixCluster(zkConnectionString, helixClusterName, false); return HelixUtils.buildHelixManager(helixInstanceName, helixClusterName, zkConnectionString); }
public SimpleKafkaSpecExecutorInstance(Config config, Optional<Logger> log) { _config = config; _log = log.isPresent() ? log.get() : LoggerFactory.getLogger(getClass()); try { _specExecutorInstanceUri = new URI(ConfigUtils.getString(config, ConfigurationKeys.SPECEXECUTOR_INSTANCE_URI_KEY, "NA")); } catch (URISyntaxException e) { throw new RuntimeException(e); } _capabilities = Maps.newHashMap(); if (config.hasPath(ConfigurationKeys.SPECEXECUTOR_INSTANCE_CAPABILITIES_KEY)) { String capabilitiesStr = config.getString(ConfigurationKeys.SPECEXECUTOR_INSTANCE_CAPABILITIES_KEY); List<String> capabilities = SPLIT_BY_COMMA.splitToList(capabilitiesStr); for (String capability : capabilities) { List<String> currentCapability = SPLIT_BY_COLON.splitToList(capability); Preconditions.checkArgument(currentCapability.size() == 2, "Only one source:destination pair is supported " + "per capability, found: " + currentCapability); _capabilities.put(currentCapability.get(0), currentCapability.get(1)); } } }
@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { // Add all job configuration properties so they are picked up by Hadoop Configuration conf = new Configuration(); for (Map.Entry<String, ConfigValue> entry : config.entrySet()) { conf.set(entry.getKey(), entry.getValue().unwrapped().toString()); } try { String stateStoreFsUri = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_FS_URI_KEY, ConfigurationKeys.LOCAL_FS_URI); FileSystem stateStoreFs = FileSystem.get(URI.create(stateStoreFsUri), conf); String stateStoreRootDir = config.getString(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY); return new FsStateStore(stateStoreFs, stateStoreRootDir, stateClass); } catch (IOException e) { throw new RuntimeException("Failed to create FsStateStore with factory", e); } } }
public CleanableHiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties jobProps, Config config) throws IOException { super(fs, clientPool, table, jobProps, config); try { this.hiveSelectionPolicy = (VersionSelectionPolicy) GobblinConstructorUtils.invokeFirstConstructor(Class.forName(ConfigUtils.getString( this.datasetConfig, SELECTION_POLICY_CLASS_KEY, DEFAULT_SELECTION_POLICY_CLASS)), ImmutableList.<Object> of( this.datasetConfig, jobProps), ImmutableList.<Object> of(this.datasetConfig), ImmutableList.<Object> of(jobProps)); log.info(String.format("Configured selection policy %s for dataset:%s with config %s", ConfigUtils.getString(this.datasetConfig, SELECTION_POLICY_CLASS_KEY, DEFAULT_SELECTION_POLICY_CLASS), datasetURN(), this.datasetConfig.root().render(ConfigRenderOptions.concise()))); this.hiveDatasetVersionFinder = (AbstractHiveDatasetVersionFinder) GobblinConstructorUtils.invokeFirstConstructor(Class.forName(ConfigUtils .getString(this.datasetConfig, VERSION_FINDER_CLASS_KEY, DEFAULT_VERSION_FINDER_CLASS)), ImmutableList .<Object> of(this.fs, this.datasetConfig), ImmutableList.<Object> of(this.fs, jobProps)); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { log.error("Failed to instantiate CleanableHiveDataset", e); throw new IllegalArgumentException(e); } this.fsCleanableHelper = new FsCleanableHelper(fs, jobProps, this.datasetConfig, log); this.shouldDeleteData = Boolean.valueOf(jobProps.getProperty(SHOULD_DELETE_DATA_KEY, SHOULD_DELETE_DATA_DEFAULT)); this.simulate = Boolean.valueOf(jobProps.getProperty(FsCleanableHelper.SIMULATE_KEY, FsCleanableHelper.SIMULATE_DEFAULT)); }
public static KafkaProducer getKafkaProducer(Properties props) { Config config = ConfigFactory.parseProperties(props); String kafkaProducerClass = ConfigUtils.getString(config, KafkaWriterConfigurationKeys.KAFKA_WRITER_PRODUCER_CLASS, ""); Properties producerProps = getProducerProperties(props); if (kafkaProducerClass.isEmpty()) { return new KafkaProducer<>(producerProps); } else { try { Class<?> producerClass = (Class<?>) Class.forName(kafkaProducerClass); KafkaProducer producer = (KafkaProducer) ConstructorUtils.invokeConstructor(producerClass, producerProps); return producer; } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) { log.error("Failed to instantiate Kafka producer " + kafkaProducerClass + " as instance of KafkaProducer.class", e); throw Throwables.propagate(e); } } }
@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { BasicDataSource basicDataSource = MysqlStateStore.newDataSource(config); String stateStoreTableName = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_DB_TABLE); boolean compressedValues = ConfigUtils.getBoolean(config, ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES); try { return new MysqlStateStore(basicDataSource, stateStoreTableName, compressedValues, stateClass); } catch (Exception e) { throw new RuntimeException("Failed to create MysqlStateStore with factory", e); } } }
protected DatasetStateStore createStateStore(Config jobConfig) throws IOException { boolean stateStoreEnabled = !jobConfig.hasPath(ConfigurationKeys.STATE_STORE_ENABLED) || jobConfig .getBoolean(ConfigurationKeys.STATE_STORE_ENABLED); String stateStoreType; if (!stateStoreEnabled) { stateStoreType = ConfigurationKeys.STATE_STORE_TYPE_NOOP; } else { stateStoreType = ConfigUtils .getString(jobConfig, ConfigurationKeys.STATE_STORE_TYPE_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_TYPE); } ClassAliasResolver<DatasetStateStore.Factory> resolver = new ClassAliasResolver<>(DatasetStateStore.Factory.class); try { DatasetStateStore.Factory stateStoreFactory = resolver.resolveClass(stateStoreType).newInstance(); return stateStoreFactory.createStateStore(jobConfig); } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new IOException(e); } }
public StreamingJobConfigurationManager(EventBus eventBus, Config config, MutableJobCatalog jobCatalog) { super(eventBus, config); this.stopTimeoutSeconds = ConfigUtils.getLong(config, GobblinClusterConfigurationKeys.STOP_TIMEOUT_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_STOP_TIMEOUT_SECONDS); this.fetchJobSpecExecutor = Executors.newSingleThreadExecutor( ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("FetchJobSpecExecutor"))); String specExecutorInstanceConsumerClassName = ConfigUtils.getString(config, GobblinClusterConfigurationKeys.SPEC_EXECUTOR_INSTANCE_CONSUMER_CLASS_KEY, GobblinClusterConfigurationKeys.DEFAULT_STREAMING_SPEC_EXECUTOR_INSTANCE_CONSUMER_CLASS); LOGGER.info("Using SpecExecutorInstanceConsumer ClassNameclass name/alias " + specExecutorInstanceConsumerClassName); try { ClassAliasResolver<SpecExecutorInstanceConsumer> aliasResolver = new ClassAliasResolver<>(SpecExecutorInstanceConsumer.class); this.specExecutorInstanceConsumer = (SpecExecutorInstanceConsumer) GobblinConstructorUtils.invokeFirstConstructor( Class.forName(aliasResolver.resolve(specExecutorInstanceConsumerClassName)), ImmutableList.<Object>of(config, jobCatalog), ImmutableList.<Object>of(config)); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new RuntimeException("Could not construct SpecExecutorInstanceConsumer " + specExecutorInstanceConsumerClassName, e); } }
@Override public FileSystem instrumentFileSystem(FileSystem fs, SharedResourcesBroker<S> broker, ConfigView<S, FileSystemKey> config) { Config metaConfig = config.getConfig(); String metadataProviderAlias = ConfigUtils.getString(metaConfig, METADATA_PROVIDER_ALIAS, ""); log.info("Metadata provider alias is: " + metadataProviderAlias); if (!metadataProviderAlias.isEmpty()) { DatasetAwareFsMetadataProvider metadataProvider = null; try { metadataProvider = (DatasetAwareFsMetadataProvider) new ClassAliasResolver<>(DatasetAwareMetadataProviderFactory.class) .resolveClass(metadataProviderAlias).newInstance().createMetadataProvider(metaConfig); } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) { log.error("Failed to create metadataProvider.", e); } if (metadataProvider != null) { return new MetadataAwareFileSystem(fs, metadataProvider); } } log.warn("No valid {} found. Will use filesystem {}.", METADATA_PROVIDER_ALIAS, fs.getClass().getName()); return fs; } }
private Kafka08ConsumerClient(Config config) { super(config); bufferSize = ConfigUtils.getInt(config, CONFIG_KAFKA_BUFFER_SIZE_BYTES, CONFIG_KAFKA_BUFFER_SIZE_BYTES_DEFAULT); clientName = ConfigUtils.getString(config, CONFIG_KAFKA_CLIENT_NAME, CONFIG_KAFKA_CLIENT_NAME_DEFAULT); fetchCorrelationId = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_REQUEST_CORRELATION_ID, CONFIG_KAFKA_FETCH_REQUEST_CORRELATION_ID_DEFAULT); fetchTopicRetries = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_TOPIC_NUM_TRIES, CONFIG_KAFKA_FETCH_TOPIC_NUM_TRIES_DEFAULT); fetchOffsetRetries = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_OFFSET_NUM_TRIES, CONFIG_KAFKA_FETCH_OFFSET_NUM_TRIES_DEFAULT); }
public HiveDataset(FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Properties properties, Config datasetConfig) { this.fs = fs; this.clientPool = clientPool; this.table = table; this.properties = properties; this.tableRootPath = PathUtils.isGlob(this.table.getDataLocation()) ? Optional.<Path> absent() : Optional.fromNullable(this.table.getDataLocation()); this.tableIdentifier = this.table.getDbName() + "." + this.table.getTableName(); this.datasetNamePattern = Optional.fromNullable(ConfigUtils.getString(datasetConfig, DATASET_NAME_PATTERN_KEY, null)); this.dbAndTable = new DbAndTable(table.getDbName(), table.getTableName()); if (this.datasetNamePattern.isPresent()) { this.logicalDbAndTable = parseLogicalDbAndTable(this.datasetNamePattern.get(), this.dbAndTable, LOGICAL_DB_TOKEN, LOGICAL_TABLE_TOKEN); } else { this.logicalDbAndTable = this.dbAndTable; } this.datasetConfig = resolveConfig(datasetConfig, dbAndTable, logicalDbAndTable); this.metricContext = Instrumented.getMetricContext(new State(properties), HiveDataset.class, Lists.<Tag<?>> newArrayList(new Tag<>(DATABASE, table.getDbName()), new Tag<>(TABLE, table.getTableName()))); }
protected KafkaOldAPI(Config config) { super(config); socketTimeoutMillis = ConfigUtils.getInt(config, CONFIG_KAFKA_SOCKET_TIMEOUT_VALUE, CONFIG_KAFKA_SOCKET_TIMEOUT_VALUE_DEFAULT); bufferSize = ConfigUtils.getInt(config, CONFIG_KAFKA_BUFFER_SIZE_BYTES, CONFIG_KAFKA_BUFFER_SIZE_BYTES_DEFAULT); clientName = ConfigUtils.getString(config, CONFIG_KAFKA_CLIENT_NAME, CONFIG_KAFKA_CLIENT_NAME_DEFAULT); fetchCorrelationId = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_REQUEST_CORRELATION_ID, CONFIG_KAFKA_FETCH_REQUEST_CORRELATION_ID_DEFAULT); fetchTimeoutMillis = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_TIMEOUT_VALUE, CONFIG_KAFKA_FETCH_TIMEOUT_VALUE_DEFAULT); fetchMinBytes = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_REQUEST_MIN_BYTES, CONFIG_KAFKA_FETCH_REQUEST_MIN_BYTES_DEFAULT); fetchTopicRetries = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_TOPIC_NUM_TRIES, CONFIG_KAFKA_FETCH_TOPIC_NUM_TRIES_DEFAULT); fetchOffsetRetries = ConfigUtils.getInt(config, CONFIG_KAFKA_FETCH_OFFSET_NUM_TRIES, CONFIG_KAFKA_FETCH_OFFSET_NUM_TRIES_DEFAULT); Preconditions.checkArgument((this.fetchTimeoutMillis < this.socketTimeoutMillis), "Kafka Source configuration error: FetchTimeout " + this.fetchTimeoutMillis + " must be smaller than SocketTimeout " + this.socketTimeoutMillis); }