public ConfigBasedTopologySpecFactory(Config config, Optional<Logger> log) { Preconditions.checkNotNull(config, "Config should not be null"); _log = log.isPresent() ? log.get() : LoggerFactory.getLogger(getClass()); _config = config; _aliasResolver = new ClassAliasResolver<>(SpecExecutorInstanceProducer.class); }
private List<Alias> getAllAliases() { ClassAliasResolver<EmbeddedGobblinCliFactory> resolver = new ClassAliasResolver<>(EmbeddedGobblinCliFactory.class); return resolver.getAliasObjects(); }
public RetentionAction(Config actionConfig, FileSystem fs, Config jobConfig) { this.versionSelectionAliasResolver = new ClassAliasResolver<>(VersionSelectionPolicy.class); this.fs = fs; this.isSimulateMode = ConfigUtils.getBoolean(jobConfig, FsCleanableHelper.SIMULATE_KEY, Boolean.valueOf(FsCleanableHelper.SIMULATE_DEFAULT)); }
/** * Return an {@link CompactionSuiteFactory} based on the configuration * @return A concrete suite factory instance. By default {@link CompactionAvroSuiteFactory} is used. */ public static CompactionSuiteFactory getCompactionSuiteFactory (State state) { try { String factoryName = state.getProp(ConfigurationKeys.COMPACTION_SUITE_FACTORY, ConfigurationKeys.DEFAULT_COMPACTION_SUITE_FACTORY); ClassAliasResolver<CompactionSuiteFactory> conditionClassAliasResolver = new ClassAliasResolver<>(CompactionSuiteFactory.class); CompactionSuiteFactory factory = conditionClassAliasResolver.resolveClass(factoryName).newInstance(); return factory; } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { throw new IllegalArgumentException(e); } } }
public SpecExecutorInstanceProducer getSpecExecutorInstanceProducer() { if (null == specExecutorInstanceProducer) { String specExecutorInstanceProducerClass = DEFAULT_SPEC_EXECUTOR_INSTANCE_PRODUCER; if (config.hasPath(SPEC_EXECUTOR_INSTANCE_PRODUCER_KEY)) { specExecutorInstanceProducerClass = config.getString(SPEC_EXECUTOR_INSTANCE_PRODUCER_KEY); } try { ClassAliasResolver<SpecExecutorInstanceProducer> _aliasResolver = new ClassAliasResolver<>(SpecExecutorInstanceProducer.class); specExecutorInstanceProducer = (SpecExecutorInstanceProducer) ConstructorUtils .invokeConstructor(Class.forName(_aliasResolver .resolve(specExecutorInstanceProducerClass)), config); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new RuntimeException(e); } } return specExecutorInstanceProducer; }
/** * Infer and construct a {@link RequestAllocator} from an input {@link Config}. */ public static <T extends Request<T>> RequestAllocator<T> inferFromConfig(RequestAllocatorConfig<T> configuration) { try { String alias = configuration.getLimitedScopeConfig().hasPath(ALLOCATOR_ALIAS_KEY) ? configuration.getLimitedScopeConfig().getString(ALLOCATOR_ALIAS_KEY) : BruteForceAllocator.Factory.class.getName(); RequestAllocator.Factory allocatorFactory = new ClassAliasResolver<>(RequestAllocator.Factory.class). resolveClass(alias).newInstance(); log.info("Using allocator factory " + allocatorFactory.getClass().getName()); return allocatorFactory.createRequestAllocator(configuration); } catch (ReflectiveOperationException roe) { throw new RuntimeException(roe); } } }
public static void main(String[] args) { ClassAliasResolver<CliApplication> resolver = new ClassAliasResolver<>(CliApplication.class); if (args.length < 1 || Sets.newHashSet("-h", "--help").contains(args[0])) { printUsage(resolver); return; } String alias = args[0]; try { CliApplication application = resolver.resolveClass(alias).newInstance(); application.run(args); } catch (ReflectiveOperationException roe) { System.err.println("Could not find an application with alias " + alias); printUsage(resolver); } catch (Throwable t) { System.out.println("Error: " + t.getMessage()); t.printStackTrace(); } }
protected KafkaReporter(Builder<?> builder, Config config) throws IOException { super(builder, config); SchemaVersionWriter versionWriter; if (config.hasPath(SCHEMA_VERSION_WRITER_TYPE)) { try { ClassAliasResolver<SchemaVersionWriter> resolver = new ClassAliasResolver<>(SchemaVersionWriter.class); Class<? extends SchemaVersionWriter> klazz = resolver.resolveClass(config.getString(SCHEMA_VERSION_WRITER_TYPE)); versionWriter = klazz.newInstance(); } catch (ReflectiveOperationException roe) { throw new IOException("Could not instantiate version writer.", roe); } } else { versionWriter = new FixedSchemaVersionWriter(); } log.info("Schema version writer: " + versionWriter.getClass().getName()); this.serializer = this.closer.register(createSerializer(versionWriter)); if (builder.kafkaPusher.isPresent()) { this.kafkaPusher = builder.kafkaPusher.get(); } else { this.kafkaPusher = this.closer.register(new KafkaPusher(builder.brokers, builder.topic)); } }
public ScheduledJobConfigurationManager(EventBus eventBus, Config config) { super(eventBus, config); this.jobSpecs = Maps.newHashMap(); this.refreshIntervalInSeconds = ConfigUtils.getLong(config, GobblinClusterConfigurationKeys.JOB_SPEC_REFRESH_INTERVAL, DEFAULT_JOB_SPEC_REFRESH_INTERVAL); this.fetchJobSpecExecutor = Executors.newSingleThreadScheduledExecutor( ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("FetchJobSpecExecutor"))); this.aliasResolver = new ClassAliasResolver<>(SpecExecutorInstanceConsumer.class); try { String specExecutorInstanceConsumerClassName = GobblinClusterConfigurationKeys.DEFAULT_SPEC_EXECUTOR_INSTANCE_CONSUMER_CLASS; if (config.hasPath(GobblinClusterConfigurationKeys.SPEC_EXECUTOR_INSTANCE_CONSUMER_CLASS_KEY)) { specExecutorInstanceConsumerClassName = config.getString(GobblinClusterConfigurationKeys.SPEC_EXECUTOR_INSTANCE_CONSUMER_CLASS_KEY); } LOGGER.info("Using SpecExecutorInstanceConsumer ClassNameclass name/alias " + specExecutorInstanceConsumerClassName); this.specExecutorInstanceConsumer = (SpecExecutorInstanceConsumer) ConstructorUtils .invokeConstructor(Class.forName(this.aliasResolver.resolve( specExecutorInstanceConsumerClassName)), config); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new RuntimeException(e); } }
protected DatasetStateStore createStateStore(Config jobConfig) throws IOException { boolean stateStoreEnabled = !jobConfig.hasPath(ConfigurationKeys.STATE_STORE_ENABLED) || jobConfig .getBoolean(ConfigurationKeys.STATE_STORE_ENABLED); String stateStoreType; if (!stateStoreEnabled) { stateStoreType = ConfigurationKeys.STATE_STORE_TYPE_NOOP; } else { stateStoreType = ConfigUtils .getString(jobConfig, ConfigurationKeys.STATE_STORE_TYPE_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_TYPE); } ClassAliasResolver<DatasetStateStore.Factory> resolver = new ClassAliasResolver<>(DatasetStateStore.Factory.class); try { DatasetStateStore.Factory stateStoreFactory = resolver.resolveClass(stateStoreType).newInstance(); return stateStoreFactory.createStateStore(jobConfig); } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new IOException(e); } }
public StateStoreBasedWatermarkStorage(State taskState) { Preconditions.checkArgument(taskState != null); Preconditions.checkArgument(!taskState.getProp(ConfigurationKeys.JOB_NAME_KEY).isEmpty()); String watermarkStateStoreType = taskState.getProp(WATERMARK_STORAGE_TYPE_KEY, WATERMARK_STORAGE_TYPE_DEFAULT); ClassAliasResolver<StateStore.Factory> resolver = new ClassAliasResolver<>(StateStore.Factory.class); StateStore.Factory stateStoreFactory; try { stateStoreFactory = resolver.resolveClass(watermarkStateStoreType).newInstance(); } catch (ClassNotFoundException cnfe) { throw new RuntimeException(cnfe); } catch (InstantiationException ie) { throw new RuntimeException(ie); } catch (IllegalAccessException iae) { throw new RuntimeException(iae); } Config config = getStateStoreConfig(taskState); _stateStore = stateStoreFactory.createStateStore(config, CheckpointableWatermarkState.class); _storeName = WATERMARK_STORAGE_PREFIX + taskState.getProp(ConfigurationKeys.JOB_NAME_KEY); log.info("State Store directory configured as : {}", config.getString(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY)); log.info("Configured the StateStoreBasedWatermarkStorage with storeName: {}", _storeName); }
private ImmutableList<RecompactionCondition> getConditionsFromProperties (Dataset dataset) { ClassAliasResolver<RecompactionConditionFactory> conditionClassAliasResolver = new ClassAliasResolver<>(RecompactionConditionFactory.class); List<String> factoryNames = dataset.jobProps().getPropAsList(MRCompactor.COMPACTION_RECOMPACT_COMBINE_CONDITIONS, MRCompactor.DEFAULT_COMPACTION_RECOMPACT_CONDITION); ImmutableList.Builder<RecompactionCondition> builder = ImmutableList.builder(); for (String factoryName : factoryNames) { try { RecompactionConditionFactory factory = GobblinConstructorUtils.invokeFirstConstructor( conditionClassAliasResolver.resolveClass(factoryName), ImmutableList.of()); RecompactionCondition condition = factory.createRecompactionCondition(dataset); builder.add(condition); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new IllegalArgumentException(e); } } return builder.build(); }
private CompactorCompletionListener getCompactionCompleteListener () { ClassAliasResolver<CompactorCompletionListenerFactory> classAliasResolver = new ClassAliasResolver<>(CompactorCompletionListenerFactory.class); String listenerName= this.state.getProp(MRCompactor.COMPACTION_COMPLETE_LISTERNER, MRCompactor.DEFAULT_COMPACTION_COMPLETE_LISTERNER); try { CompactorCompletionListenerFactory factory = GobblinConstructorUtils.invokeFirstConstructor( classAliasResolver.resolveClass(listenerName), ImmutableList.of()); return factory.createCompactorCompactionListener(this.state); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new IllegalArgumentException(e); } }
public StreamingJobConfigurationManager(EventBus eventBus, Config config, MutableJobCatalog jobCatalog) { super(eventBus, config); this.stopTimeoutSeconds = ConfigUtils.getLong(config, GobblinClusterConfigurationKeys.STOP_TIMEOUT_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_STOP_TIMEOUT_SECONDS); this.fetchJobSpecExecutor = Executors.newSingleThreadExecutor( ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("FetchJobSpecExecutor"))); String specExecutorInstanceConsumerClassName = ConfigUtils.getString(config, GobblinClusterConfigurationKeys.SPEC_EXECUTOR_INSTANCE_CONSUMER_CLASS_KEY, GobblinClusterConfigurationKeys.DEFAULT_STREAMING_SPEC_EXECUTOR_INSTANCE_CONSUMER_CLASS); LOGGER.info("Using SpecExecutorInstanceConsumer ClassNameclass name/alias " + specExecutorInstanceConsumerClassName); try { ClassAliasResolver<SpecExecutorInstanceConsumer> aliasResolver = new ClassAliasResolver<>(SpecExecutorInstanceConsumer.class); this.specExecutorInstanceConsumer = (SpecExecutorInstanceConsumer) GobblinConstructorUtils.invokeFirstConstructor( Class.forName(aliasResolver.resolve(specExecutorInstanceConsumerClassName)), ImmutableList.<Object>of(config, jobCatalog), ImmutableList.<Object>of(config)); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new RuntimeException("Could not construct SpecExecutorInstanceConsumer " + specExecutorInstanceConsumerClassName, e); } }
@Override public FileSystem instrumentFileSystem(FileSystem fs, SharedResourcesBroker<S> broker, ConfigView<S, FileSystemKey> config) { Config metaConfig = config.getConfig(); String metadataProviderAlias = ConfigUtils.getString(metaConfig, METADATA_PROVIDER_ALIAS, ""); log.info("Metadata provider alias is: " + metadataProviderAlias); if (!metadataProviderAlias.isEmpty()) { DatasetAwareFsMetadataProvider metadataProvider = null; try { metadataProvider = (DatasetAwareFsMetadataProvider) new ClassAliasResolver<>(DatasetAwareMetadataProviderFactory.class) .resolveClass(metadataProviderAlias).newInstance().createMetadataProvider(metaConfig); } catch (ClassNotFoundException | IllegalAccessException | InstantiationException e) { log.error("Failed to create metadataProvider.", e); } if (metadataProvider != null) { return new MetadataAwareFileSystem(fs, metadataProvider); } } log.warn("No valid {} found. Will use filesystem {}.", METADATA_PROVIDER_ALIAS, fs.getClass().getName()); return fs; } }
public Orchestrator(Config config, Optional<TopologyCatalog> topologyCatalog, Optional<Logger> log, boolean instrumentationEnabled) { _log = log.isPresent() ? log.get() : LoggerFactory.getLogger(getClass()); if (instrumentationEnabled) { this.metricContext = Instrumented.getMetricContext(ConfigUtils.configToState(config), IdentityFlowToJobSpecCompiler.class); this.flowOrchestrationSuccessFulMeter = Optional.of(this.metricContext.meter(ServiceMetricNames.FLOW_ORCHESTRATION_SUCCESSFUL_METER)); this.flowOrchestrationFailedMeter = Optional.of(this.metricContext.meter(ServiceMetricNames.FLOW_ORCHESTRATION_FAILED_METER)); this.flowOrchestrationTimer = Optional.<Timer>of(this.metricContext.timer(ServiceMetricNames.FLOW_ORCHESTRATION_TIMER)); } else { this.metricContext = null; this.flowOrchestrationSuccessFulMeter = Optional.absent(); this.flowOrchestrationFailedMeter = Optional.absent(); this.flowOrchestrationTimer = Optional.absent(); } this.aliasResolver = new ClassAliasResolver<>(SpecCompiler.class); this.topologyCatalog = topologyCatalog; try { String specCompilerClassName = ServiceConfigKeys.DEFAULT_GOBBLIN_SERVICE_FLOWCOMPILER_CLASS; if (config.hasPath(ServiceConfigKeys.GOBBLIN_SERVICE_FLOWCOMPILER_CLASS_KEY)) { specCompilerClassName = config.getString(ServiceConfigKeys.GOBBLIN_SERVICE_FLOWCOMPILER_CLASS_KEY); } _log.info("Using specCompiler class name/alias " + specCompilerClassName); this.specCompiler = (SpecCompiler) ConstructorUtils.invokeConstructor(Class.forName(this.aliasResolver.resolve( specCompilerClassName)), config); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new RuntimeException(e); } }
private RecompactionCondition createRecompactionCondition () { ClassAliasResolver<RecompactionConditionFactory> conditionClassAliasResolver = new ClassAliasResolver<>(RecompactionConditionFactory.class); String factoryName = this.dataset.jobProps().getProp(MRCompactor.COMPACTION_RECOMPACT_CONDITION, MRCompactor.DEFAULT_COMPACTION_RECOMPACT_CONDITION); try { RecompactionConditionFactory factory = GobblinConstructorUtils.invokeFirstConstructor( conditionClassAliasResolver.resolveClass(factoryName), ImmutableList.of()); return factory.createRecompactionCondition(dataset); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException | InstantiationException | ClassNotFoundException e) { throw new IllegalArgumentException(e); } }
/** * Obtain a client factory * @param state job state * @return a factory which creates {@link AuditCountClient}. * If no factory is set or an error occurred, a {@link EmptyAuditCountClientFactory} is * returned which creates a <code>null</code> {@link AuditCountClient} */ private static AuditCountClientFactory getClientFactory (State state) { if (!state.contains(AuditCountClientFactory.AUDIT_COUNT_CLIENT_FACTORY)) { return new EmptyAuditCountClientFactory (); } try { String factoryName = state.getProp(AuditCountClientFactory.AUDIT_COUNT_CLIENT_FACTORY); ClassAliasResolver<AuditCountClientFactory> conditionClassAliasResolver = new ClassAliasResolver<>(AuditCountClientFactory.class); AuditCountClientFactory factory = conditionClassAliasResolver.resolveClass(factoryName).newInstance(); return factory; } catch (Exception e) { throw new RuntimeException(e); } }
private DatasetStateStore getDatasetStateStore() throws IOException { if (this.stateStore == null) { ClassAliasResolver<DatasetStateStore.Factory> resolver = new ClassAliasResolver<>(DatasetStateStore.Factory.class); String stateStoreType = this.props.getProp(ConfigurationKeys.STATE_STORE_TYPE_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_TYPE); try { DatasetStateStore.Factory stateStoreFactory = resolver.resolveClass(stateStoreType).newInstance(); this.stateStore = stateStoreFactory.createStateStore(ConfigFactory.parseProperties(props.getProperties())); } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new IOException(e); } } return this.stateStore; }
private void initRequestAllocator (State state) { try { ResourceEstimator estimator = GobblinConstructorUtils.<ResourceEstimator>invokeLongestConstructor( new ClassAliasResolver(ResourceEstimator.class).resolveClass(state.getProp(ConfigurationKeys.COMPACTION_ESTIMATOR, SimpleDatasetRequest.SimpleDatasetCountEstimator.class.getName()))); RequestAllocatorConfig.Builder<SimpleDatasetRequest> configBuilder = RequestAllocatorConfig.builder(estimator).allowParallelization(1).withLimitedScopeConfig(ConfigBuilder.create() .loadProps(state.getProperties(), ConfigurationKeys.COMPACTION_PRIORITIZATION_PREFIX).build()); if (!state.contains(ConfigurationKeys.COMPACTION_PRIORITIZER_ALIAS)) { allocator = new GreedyAllocator<>(configBuilder.build()); return; } Comparator<SimpleDatasetRequest> prioritizer = GobblinConstructorUtils.<Comparator>invokeLongestConstructor( new ClassAliasResolver(Comparator.class).resolveClass(state.getProp(ConfigurationKeys.COMPACTION_PRIORITIZER_ALIAS)), state); configBuilder.withPrioritizer(prioritizer); if (prioritizer instanceof HierarchicalPrioritizer) { allocator = new HierarchicalAllocator.Factory().createRequestAllocator(configBuilder.build()); } else { allocator = RequestAllocatorUtils.inferFromConfig(configBuilder.build()); } } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new RuntimeException("Cannot initialize allocator", e); } }