@Alias("fs") public class FsDatasetStateStoreFactory implements DatasetStateStore.Factory { @Override public DatasetStateStore<JobState.DatasetState> createStateStore(Config config) { try { return FsDatasetStateStore.createStateStore(config, FsDatasetStateStore.class.getName()); } catch (Exception e) { throw new RuntimeException("Failed to create FsDatasetStateStore with factory", e); } } }
/** * A {@link Dataset} that can be registered in Hive. */ @Alpha public interface HiveRegistrableDataset extends Dataset { /** * Get a list of {@link HiveSpec}s for this dataset, which can be used by {@link org.apache.gobblin.hive.HiveRegister} * to register this dataset in Hive. */ public List<HiveSpec> getHiveSpecs() throws IOException; }
/** * NoOp dynamic config generator that returns an empty {@link Config} */ @Alias("noop") public class NoopDynamicConfigGenerator implements DynamicConfigGenerator { public NoopDynamicConfigGenerator() { } public Config generateDynamicConfig(Config config) { return ConfigFactory.empty(); } }
/** * A dummy class representing an ApplicationMaster shutdown request to be used with a * {@link com.google.common.eventbus.EventBus}. * * @author Yinan Li */ @Alpha public class ClusterManagerShutdownRequest { }
@Alias("noop") public static class Factory implements DatasetStateStore.Factory { @Override public DatasetStateStore<JobState.DatasetState> createStateStore(Config config) { // dummy root dir for noop state store Config config2 = config.withValue(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, ConfigValueFactory.fromAnyRef("")); return FsDatasetStateStore.createStateStore(config2, NoopDatasetStateStore.class.getName()); } }
/** * A {@link HiveSpec} with a set of {@link Predicate}s. If any of the {@link Predicate}s returns false, * the Hive registration will be skipped. */ @Alpha public interface HiveSpecWithPredicates extends HiveSpec { /** * A {@link Collection} of {@link Predicate}s. If any of the {@link Predicate}s returns false, * the Hive registration will be skipped. */ public Collection<Predicate<HiveRegister>> getPredicates(); }
/** * A factory for the default implementation of the admin web server */ @Alias(value="default") public class DefaultAdminWebServerFactory implements AdminWebServerFactory { /** {@inheritDoc} */ @Override public Service createInstance(Properties config, URI executionInfoServerURI) { return new AdminWebServer(config, executionInfoServerURI); } }
/** * An interface for implementing streaming / continuous extractors */ @Alpha public interface StreamingExtractor<S, D> extends Extractor<S, D> { /** * Initialize the extractor to be ready to read records * @param watermarkStorage : watermark storage to retrieve previously committed watermarks * @throws IOException : typically if there was a failure in retrieving watermarks */ void start(WatermarkStorage watermarkStorage) throws IOException; }
@Alias(value = "PoolBasedLimiter") public static class Factory implements LimiterFactory { public static final String POOL_SIZE_KEY = "poolSize"; @Override public Limiter buildLimiter(Config config) { if (!config.hasPath(POOL_SIZE_KEY)) { throw new IllegalArgumentException("Missing key " + POOL_SIZE_KEY); } return new PoolBasedLimiter(config.getInt(POOL_SIZE_KEY)); } }
/** * An {@link Exception} thrown by an {@link ApplicationLauncher}. */ @Alpha public class ApplicationException extends Exception { private static final long serialVersionUID = -7131035635096992762L; public ApplicationException(String message, Throwable cause) { super(message, cause); } }
@Alias("RecompactBasedOnDuration") public static class Factory implements RecompactionConditionFactory { @Override public RecompactionCondition createRecompactionCondition (Dataset dataset) { return new RecompactionConditionBasedOnDuration (dataset); } }
@Alias(value = "qps") public static class Factory implements LimiterFactory { public static final String QPS_KEY = "qps"; @Override public Limiter buildLimiter(Config config) { if (!config.hasPath(QPS_KEY)) { throw new RuntimeException("Missing key " + QPS_KEY); } return new RateBasedLimiter(config.getLong(QPS_KEY)); } }
@Alias(FACTORY_ALIAS) public static class Factory implements ThrottlingPolicyFactory.SpecificPolicyFactory { @Override public ThrottlingPolicy createPolicy(SharedLimiterKey key, SharedResourcesBroker<ThrottlingServerScopes> broker, Config config) { Preconditions.checkArgument(config.hasPath(COUNT_KEY), "Missing key " + COUNT_KEY); return new CountBasedPolicy(config.getLong(COUNT_KEY)); } }
@Alias(FACTORY_ALIAS) public static class Factory implements ThrottlingPolicyFactory.SpecificPolicyFactory { @Override public ThrottlingPolicy createPolicy(SharedLimiterKey key, SharedResourcesBroker<ThrottlingServerScopes> broker, Config config) { return new NoopPolicy(); } }
@Alias("noop") public class NoopEventMetadataGenerator implements EventMetadataGenerator{ public Map<String, String> getMetadata(JobContext jobContext, EventName eventName) { return ImmutableMap.of(); } }
@Alias("fake1") static class FakePluginFactory1 implements GobblinInstancePluginFactory { @Override public GobblinInstancePlugin createPlugin(GobblinInstanceDriver instance) { return new FakePlugin(instance); } }
/** * Test hive registration failure */ @Alias("HiveRegistrationFailureFactory") public static class HiveRegistrationFailureFactory extends CompactionAvroSuiteFactory { public TestCompactionSuites.HiveRegistrationCompactionSuite createSuite (State state) { return new TestCompactionSuites.HiveRegistrationCompactionSuite(state); } } }
@Alias(value = "MockSink") public class MockSink implements AuditSink { public MockSink(Config config, ValueAuditRuntimeMetadata auditMetadata) {} @Override public void close() throws IOException {} @Override public void write(GenericRecord record) throws IOException {} }
@Alias("TestDistributedExecutionLauncherBuilder") public static class Builder extends GobblinHelixDistributeJobExecutionLauncher.Builder { public TestDistributedExecutionLauncher build() throws Exception { return new TestDistributedExecutionLauncher(this); } } }
@Alias("fake2") static class FakePluginFactory2 implements GobblinInstancePluginFactory { @Override public GobblinInstancePlugin createPlugin(GobblinInstanceDriver instance) { return new FakePlugin(instance); } }