private void runDistcp(Path from, Path to) throws Exception { info("sourcePath: " + from + ", destinationPath: " + to); EmbeddedGobblin embeddedGobblin = new EmbeddedGobblinDistcp(from, to).mrMode(); // Used for global throttling" embeddedGobblin.distributeJar("lib/*"); for (Map.Entry<String, String> entry : this.props.entrySet()) { if (entry.getKey() != null && (entry.getKey()).startsWith(ATTR_PREFIX)) { String key = (entry.getKey()).substring(ATTR_PREFIX.length()); embeddedGobblin.setConfiguration(key, entry.getValue()); } } JobExecutionResult result = embeddedGobblin.run(); if (!result.isSuccessful()) { throw new RuntimeException("Distcp job failed!", result.getErrorCause()); } }
@Override @NotOnCli public EmbeddedGobblin setTemplate(String templateURI) throws URISyntaxException, SpecNotFoundException, JobTemplate.TemplateException { return super.setTemplate(templateURI); } }
/** * Enable state store. */ public EmbeddedGobblin useStateStore(String rootDir) { this.setConfiguration(ConfigurationKeys.STATE_STORE_ENABLED, "true"); this.setConfiguration(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, rootDir); return this; }
@CliObjectSupport(argumentNames = {"jobName"}) public EmbeddedGobblin(String name) { HadoopUtils.addGobblinSite(); this.specBuilder = new JobSpec.Builder(name); this.userConfigMap = Maps.newHashMap(); this.builtConfigMap = Maps.newHashMap(); this.sysConfigOverrides = Maps.newHashMap(); this.defaultSysConfig = getDefaultSysConfig(); this.distributedJars = Maps.newHashMap(); loadCoreGobblinJarsToDistributedJars(); this.distributeJarsFunction = new Runnable() { @Override public void run() { // NOOP } }; }
embeddedGobblin.run(); } catch (InterruptedException | TimeoutException | ExecutionException exc) { throw new RuntimeException("Failed to run Gobblin job.", exc);
/** * Specify that the input jar should be added to workers' classpath on distributed mode. */ public EmbeddedGobblin distributeJar(String jarPath) { return distributeJarWithPriority(jarPath, 0); }
distributeJarByClassWithPriority(State.class, 0); distributeJarByClassWithPriority(ConstructState.class, 0); distributeJarByClassWithPriority(InstrumentedExtractorBase.class, 0); distributeJarByClassWithPriority(MetricContext.class, 0); distributeJarByClassWithPriority(GobblinMetrics.class, 0); distributeJarByClassWithPriority(FsStateStore.class, 0); distributeJarByClassWithPriority(Task.class, 0); distributeJarByClassWithPriority(PathUtils.class, 0); distributeJarByClassWithPriority(ReadableInstant.class, 0); distributeJarByClassWithPriority(Escaper.class, -10); // Escaper was added in guava 15, so we use it to identify correct jar distributeJarByClassWithPriority(MetricFilter.class, 0); distributeJarByClassWithPriority(DataTemplate.class, 0); distributeJarByClassWithPriority(ClassUtils.class, 0); distributeJarByClassWithPriority(SchemaBuilder.class, 0); distributeJarByClassWithPriority(RetryListener.class, 0);
@Override public void run() { // Add jars needed at runtime to the sys config so MR job launcher will add them to distributed cache. EmbeddedGobblin.this.sysConfigOverrides.put(ConfigurationKeys.JOB_JAR_FILES_KEY, Joiner.on(",").join(getPrioritizedDistributedJars())); } };
new SimpleGobblinInstanceEnvironment("EmbeddedGobblinInstance", this.useLog, getSysConfig());
/** * Set a {@link JobTemplate} to use. */ public EmbeddedGobblin setTemplate(String templateURI) throws URISyntaxException, SpecNotFoundException, JobTemplate.TemplateException { return setTemplate(new PackagedTemplatesJobCatalogDecorator().getTemplate(new URI(templateURI))); }
/** * Manually set a key-value pair in the job configuration. Input is of the form <key>:<value> */ public EmbeddedGobblin setConfiguration(String keyValue) { List<String> split = KEY_VALUE_SPLITTER.splitToList(keyValue); if (split.size() != 2) { throw new RuntimeException("Cannot parse " + keyValue + ". Expected <key>:<value>."); } return setConfiguration(split.get(0), split.get(1)); }
/** * Specify that the input jar should be added to workers' classpath on distributed mode. Jars with lower priority value * will appear first in the classpath. Default priority is 0. */ public EmbeddedGobblin distributeJarByClassWithPriority(Class<?> klazz, int priority) { return distributeJarWithPriority(ClassUtil.findContainingJar(klazz), priority); }