public SelectAfterTimeBasedPolicy(Properties props) { this(ConfigUtils.propertiesToConfig(props)); }
public TimeBasedRetentionPolicy(Properties props) { this(ConfigUtils.propertiesToConfig(props)); }
public SelectBeforeTimeBasedPolicy(Properties props) { super(ConfigUtils.propertiesToConfig(props)); }
/** * Convert a given {@link Properties} to a {@link Config} instance. * * <p> * This method will throw an exception if (1) the {@link Object#toString()} method of any two keys in the * {@link Properties} objects returns the same {@link String}, or (2) if any two keys are prefixes of one another, * see the Java Docs of {@link ConfigFactory#parseMap(Map)} for more details. * </p> * * @param properties the given {@link Properties} instance * @return a {@link Config} instance */ public static Config propertiesToConfig(Properties properties) { return propertiesToConfig(properties, Optional.<String>absent()); }
private void readObject(java.io.ObjectInputStream stream) throws IOException, ClassNotFoundException { uri = (URI) stream.readObject(); version = (String) stream.readObject(); description = (String) stream.readObject(); templateURI = Optional.fromNullable((URI) stream.readObject()); configAsProperties = (Properties) stream.readObject(); config = ConfigUtils.propertiesToConfig(configAsProperties); } }
/** * Builds a {@link OutputStreamReporter} with the given properties. * * @return a {@link OutputStreamReporter} */ public OutputStreamReporter build(Properties props) { return new OutputStreamReporter(this, ConfigUtils.propertiesToConfig(props, Optional.of(ConfigurationKeys.METRICS_CONFIGURATIONS_PREFIX))); } }
public static Optional<Config> getOptionalRuntimeConfig(Properties properties) { Properties runtimeConfigProps = new Properties(); for (Map.Entry<Object, Object> entry : properties.entrySet()) { if (entry.getKey().toString().startsWith(ConfigurationKeys.CONFIG_RUNTIME_PREFIX)) { runtimeConfigProps.put(entry.getKey().toString().replace(ConfigurationKeys.CONFIG_RUNTIME_PREFIX, ""), entry.getValue().toString()); } } if (runtimeConfigProps.size() == 0) { return Optional.<Config>absent(); } Config config = ConfigUtils.propertiesToConfig(runtimeConfigProps); return Optional.fromNullable(config); } }
public static DefaultConfigurableImpl createFromProperties(Properties srcConfig) { return new DefaultConfigurableImpl(ConfigUtils.propertiesToConfig(srcConfig), srcConfig); }
/** * Builds and returns {@link KafkaReporter}. * * @param brokers string of Kafka brokers * @param topic topic to send metrics to * @return KafkaReporter */ public KafkaReporter build(String brokers, String topic, Properties props) throws IOException { this.brokers = brokers; this.topic = topic; return new KafkaReporter(this, ConfigUtils.propertiesToConfig(props, Optional.of(ConfigurationKeys.METRICS_CONFIGURATIONS_PREFIX))); } }
/** * Builds and returns {@link KafkaAvroReporter}. * * @param brokers string of Kafka brokers * @param topic topic to send metrics to * @return KafkaAvroReporter */ public KafkaAvroReporter build(String brokers, String topic, Properties props) throws IOException { this.brokers = brokers; this.topic = topic; return new KafkaAvroReporter(this, ConfigUtils.propertiesToConfig(props, Optional.of(ConfigurationKeys.METRICS_CONFIGURATIONS_PREFIX))); } }
@Override public List<WorkUnit> getWorkunits(SourceState state) { Config rootCfg = ConfigUtils.propertiesToConfig(state.getProperties()); Config cfg = rootCfg.hasPath(CONFIG_NAMESPACE) ? rootCfg.getConfig(CONFIG_NAMESPACE) : ConfigFactory.empty(); int numHellos = cfg.hasPath(NUM_HELLOS_KEY) ? cfg.getInt(NUM_HELLOS_KEY) : DEFAULT_NUM_HELLOS; Extract extract = new Extract(TableType.APPEND_ONLY, HelloWorldSource.class.getPackage().getName(), HelloWorldSource.class.getSimpleName()); List<WorkUnit> wus = new ArrayList<>(numHellos); for (int i = 1; i <= numHellos; ++i) { WorkUnit wu = new WorkUnit(extract); wu.setProp(HELLO_ID_FULL_KEY, i); wus.add(wu); } return wus; }
/** * Load a {@link Properties} compatible path using fallback as fallback. * @return The {@link Config} in path with fallback as fallback. * @throws IOException */ private Config loadJavaPropsWithFallback(Path propertiesPath, Config fallback) throws IOException { PropertiesConfiguration propertiesConfiguration = new PropertiesConfiguration(); try (InputStreamReader inputStreamReader = new InputStreamReader(this.fs.open(propertiesPath), Charsets.UTF_8)) { propertiesConfiguration.load(inputStreamReader); Config configFromProps = ConfigUtils.propertiesToConfig(ConfigurationConverter.getProperties(propertiesConfiguration)); return ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, PathUtils.getPathWithoutSchemeAndAuthority(propertiesPath).toString())) .withFallback(configFromProps) .withFallback(fallback); } catch (ConfigurationException ce) { throw new IOException(ce); } }
private static Properties resolveTemplate(Properties jobProps) throws IOException { try { if (jobProps.containsKey(ConfigurationKeys.JOB_TEMPLATE_PATH)) { Config jobConfig = ConfigUtils.propertiesToConfig(jobProps); Properties resolvedProps = ConfigUtils.configToProperties((ResourceBasedJobTemplate .forResourcePath(jobProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH), new PackagedTemplatesJobCatalogDecorator())) .getResolvedConfig(jobConfig)); return resolvedProps; } else { return jobProps; } } catch (JobTemplate.TemplateException | SpecNotFoundException | URISyntaxException exc) { throw new IOException(exc); } } }
/** * Load job configuration from job configuration files stored in general file system, * located by Path * @param sysProps Gobblin framework configuration properties * @return a list of job configurations in the form of {@link java.util.Properties} */ public static List<Properties> loadGenericJobConfigs(Properties sysProps) throws ConfigurationException, IOException { Path rootPath = new Path(sysProps.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY)); PullFileLoader loader = new PullFileLoader(rootPath, rootPath.getFileSystem(new Configuration()), getJobConfigurationFileExtensions(sysProps), PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS); Config sysConfig = ConfigUtils.propertiesToConfig(sysProps); Collection<Config> configs = loader.loadPullFilesRecursively(rootPath, sysConfig, true); List<Properties> jobConfigs = Lists.newArrayList(); for (Config config : configs) { try { jobConfigs.add(resolveTemplate(ConfigUtils.configToProperties(config))); } catch (IOException ioe) { LOGGER.error("Could not parse job config at " + ConfigUtils.getString(config, ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, "Unknown path"), ioe); } } return jobConfigs; }
/** * Load a given job configuration file from a general file system. * * @param sysProps Gobblin framework configuration properties * @param jobConfigPath job configuration file to be loaded * @param jobConfigPathDir root job configuration file directory * @return a job configuration in the form of {@link java.util.Properties} */ public static Properties loadGenericJobConfig(Properties sysProps, Path jobConfigPath, Path jobConfigPathDir) throws ConfigurationException, IOException { PullFileLoader loader = new PullFileLoader(jobConfigPathDir, jobConfigPathDir.getFileSystem(new Configuration()), getJobConfigurationFileExtensions(sysProps), PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS); Config sysConfig = ConfigUtils.propertiesToConfig(sysProps); Config config = loader.loadPullFile(jobConfigPath, sysConfig, true); return resolveTemplate(ConfigUtils.configToProperties(config)); }
public AzkabanGobblinYarnAppLauncher(String jobId, Properties props) throws IOException { super(jobId, LOGGER); Config gobblinConfig = ConfigUtils.propertiesToConfig(props); this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(gobblinConfig, new YarnConfiguration()); }
/** * Load job configurations from job configuration files affected by changes to the given common properties file. * From a general file system. * @param sysProps Gobblin framework configuration properties * @param commonPropsPath the path of common properties file with changes * @param jobConfigPathDir the path for root job configuration file directory * @return a list of job configurations in the form of {@link java.util.Properties} */ public static List<Properties> loadGenericJobConfigs(Properties sysProps, Path commonPropsPath, Path jobConfigPathDir) throws ConfigurationException, IOException { PullFileLoader loader = new PullFileLoader(jobConfigPathDir, jobConfigPathDir.getFileSystem(new Configuration()), getJobConfigurationFileExtensions(sysProps), PullFileLoader.DEFAULT_HOCON_PULL_FILE_EXTENSIONS); Config sysConfig = ConfigUtils.propertiesToConfig(sysProps); Collection<Config> configs = loader.loadPullFilesRecursively(commonPropsPath.getParent(), sysConfig, true); List<Properties> jobConfigs = Lists.newArrayList(); for (Config config : configs) { try { jobConfigs.add(resolveTemplate(ConfigUtils.configToProperties(config))); } catch (IOException ioe) { LOGGER.error("Could not parse job config at " + ConfigUtils.getString(config, ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, "Unknown path"), ioe); } } return jobConfigs; }
private void initWithSelectionPolicy(Config config, Properties jobProps) { String selectionPolicyKey = StringUtils.substringAfter(SELECTION_POLICY_CLASS_KEY, CONFIGURATION_KEY_PREFIX); String versionFinderKey = StringUtils.substringAfter(VERSION_FINDER_CLASS_KEY, CONFIGURATION_KEY_PREFIX); Preconditions.checkArgument( config.hasPath(versionFinderKey), String.format("Version finder class is required at %s in config %s", versionFinderKey, config.root().render(ConfigRenderOptions.concise()))); VersionFinderAndPolicyBuilder<T> builder = VersionFinderAndPolicy.builder(); builder.versionFinder(createVersionFinder(config.getString(versionFinderKey), config, jobProps)); if (config.hasPath(selectionPolicyKey)) { builder.versionSelectionPolicy(createSelectionPolicy( ConfigUtils.getString(config, selectionPolicyKey, SelectNothingPolicy.class.getName()), config, jobProps)); } for (Class<? extends RetentionActionFactory> factoryClass : RETENTION_ACTION_TYPES) { try { RetentionActionFactory factory = factoryClass.newInstance(); if (factory.canCreateWithConfig(config)) { builder.retentionAction((RetentionAction) factory.createRetentionAction(config, this.fs, ConfigUtils.propertiesToConfig(jobProps))); } } catch (InstantiationException | IllegalAccessException e) { Throwables.propagate(e); } } this.versionFindersAndPolicies.add(builder.build()); }
/** * Create a KafkaWrapper based on the given type of Kafka API and list of Kafka brokers. * * @param state A {@link State} object that should contain a list of comma separated Kafka brokers * in property "kafka.brokers". It may optionally specify whether to use the new Kafka API by setting * use.new.kafka.api=true. */ public static KafkaWrapper create(State state) { Preconditions.checkNotNull(state.getProp(ConfigurationKeys.KAFKA_BROKERS), "Need to specify at least one Kafka broker."); KafkaWrapper.Builder builder = new KafkaWrapper.Builder(); if (state.getPropAsBoolean(USE_NEW_KAFKA_API, DEFAULT_USE_NEW_KAFKA_API)) { builder = builder.withNewKafkaAPI(); } Config config = ConfigUtils.propertiesToConfig(state.getProperties()); return builder.withBrokers(state.getPropAsList(ConfigurationKeys.KAFKA_BROKERS)) .withConfig(config) .build(); }
@Override public DataWriter build() throws IOException { State state = this.destination.getProperties(); Properties taskProps = state.getProperties(); Config config = ConfigUtils.propertiesToConfig(taskProps); long commitTimeoutMillis = ConfigUtils.getLong(config, BatchedEventhubDataWriter.COMMIT_TIMEOUT_MILLIS_CONFIG, BatchedEventhubDataWriter.COMMIT_TIMEOUT_MILLIS_DEFAULT); long commitStepWaitTimeMillis = ConfigUtils.getLong(config, BatchedEventhubDataWriter.COMMIT_STEP_WAIT_TIME_CONFIG, BatchedEventhubDataWriter.COMMIT_STEP_WAIT_TIME_DEFAULT); double failureAllowance = ConfigUtils.getDouble(config, BatchedEventhubDataWriter.FAILURE_ALLOWANCE_PCT_CONFIG, BatchedEventhubDataWriter.FAILURE_ALLOWANCE_PCT_DEFAULT) / 100.0; return AsyncWriterManager.builder() .config(config) .commitTimeoutMillis(commitTimeoutMillis) .commitStepWaitTimeInMillis(commitStepWaitTimeMillis) .failureAllowanceRatio(failureAllowance) .retriesEnabled(false) .asyncDataWriter(getAsyncDataWriter(taskProps)).maxOutstandingWrites(10000) .build(); } }