private void runDistcp(Path from, Path to) throws Exception { info("sourcePath: " + from + ", destinationPath: " + to); EmbeddedGobblin embeddedGobblin = new EmbeddedGobblinDistcp(from, to).mrMode(); // Used for global throttling" embeddedGobblin.distributeJar("lib/*"); for (Map.Entry<String, String> entry : this.props.entrySet()) { if (entry.getKey() != null && (entry.getKey()).startsWith(ATTR_PREFIX)) { String key = (entry.getKey()).substring(ATTR_PREFIX.length()); embeddedGobblin.setConfiguration(key, entry.getValue()); } } JobExecutionResult result = embeddedGobblin.run(); if (!result.isSuccessful()) { throw new RuntimeException("Distcp job failed!", result.getErrorCause()); } }
/** * Enable state store. */ public EmbeddedGobblin useStateStore(String rootDir) { this.setConfiguration(ConfigurationKeys.STATE_STORE_ENABLED, "true"); this.setConfiguration(ConfigurationKeys.STATE_STORE_ROOT_DIR_KEY, rootDir); return this; }
/** * Manually set a key-value pair in the job configuration. Input is of the form <key>:<value> */ public EmbeddedGobblin setConfiguration(String keyValue) { List<String> split = KEY_VALUE_SPLITTER.splitToList(keyValue); if (split.size() != 2) { throw new RuntimeException("Cannot parse " + keyValue + ". Expected <key>:<value>."); } return setConfiguration(split.get(0), split.get(1)); }