private void runDistcp(Path from, Path to) throws Exception { info("sourcePath: " + from + ", destinationPath: " + to); EmbeddedGobblin embeddedGobblin = new EmbeddedGobblinDistcp(from, to).mrMode(); // Used for global throttling" embeddedGobblin.distributeJar("lib/*"); for (Map.Entry<String, String> entry : this.props.entrySet()) { if (entry.getKey() != null && (entry.getKey()).startsWith(ATTR_PREFIX)) { String key = (entry.getKey()).substring(ATTR_PREFIX.length()); embeddedGobblin.setConfiguration(key, entry.getValue()); } } JobExecutionResult result = embeddedGobblin.run(); if (!result.isSuccessful()) { throw new RuntimeException("Distcp job failed!", result.getErrorCause()); } }
private void submitTrackingEvent(Spec spec, String operType) { submitTrackingEvent(spec.getUri(), spec.getVersion(), operType); }
private void submitTrackingEvent(JobSpec job, String operType) { submitTrackingEvent(job.getUri(), job.getVersion(), operType); }
/** * {@inheritDoc} * * NOTE: For this callback only conditions on the URI and version will be used. * */ @Override public void onDeleteJob(URI deletedJobURI, String deletedJobVersion) { JobSpec fakeJobSpec = JobSpec.builder(deletedJobURI).withVersion(deletedJobVersion).build(); if (this.filter.apply(fakeJobSpec)) { this.delegate.onDeleteJob(deletedJobURI, deletedJobVersion); } }
public TopologySpec build() { Preconditions.checkNotNull(this.uri); Preconditions.checkNotNull(this.version); return new TopologySpec(getURI(), getVersion(), getDescription(), getConfig(), getConfigAsProperties(), getSpecExceutorInstanceProducer()); }
public FSSpecStore(GobblinInstanceEnvironment env, SpecSerDe specSerDe, Optional<Logger> log) throws IOException { this(env.getSysConfig().getConfig(), specSerDe, log); }
@Override public void onJobLaunch(JobExecutionDriver jobDriver) { if (this.filter.apply(jobDriver.getJobExecutionState().getJobSpec())) { this.delegate.onJobLaunch(jobDriver); } }
@Override public void onJobScheduled(JobSpecSchedule jobSchedule) { if (this.filter.apply(jobSchedule.getJobSpec())) { this.delegate.onJobScheduled(jobSchedule); } }
/** {@inheritDoc} */ @Override public void onMetadataChange(JobExecutionState state, String key, Object oldValue, Object newValue) { if (this.filter.apply(state.getJobSpec())) { this.delegate.onMetadataChange(state, key, oldValue, newValue); } }
/** {@inheritDoc} */ @Override public void unregisterJobLifecycleListener(JobLifecycleListener listener) { _jobSchedulerDelegate.unregisterJobSpecSchedulerListener(listener); _jobCatalogDelegate.removeListener(listener); _dispatcher.removeListener(listener); }
/** {@inheritDoc} */ @Override public void registerJobLifecycleListener(JobLifecycleListener listener) { _dispatcher.addListener(listener); _jobCatalogDelegate.addListener(listener); _jobSchedulerDelegate.registerJobSpecSchedulerListener(listener); }
@Override public void registerWeakJobLifecycleListener(JobLifecycleListener listener) { _dispatcher.addWeakListener(listener); _jobCatalogDelegate.registerWeakJobCatalogListener(listener); _jobSchedulerDelegate.registerWeakJobSpecSchedulerListener(listener); }
@Override public Void apply(JobExecutionStateListener input) { input.onStageTransition(this.state, this.previousStage, this.newStage); return null; } }
@Override public Void apply(SpecCatalogListener listener) { listener.onAddSpec(_addedSpec); return null; } }
/** {@inheritDoc} */ @Override public void put(JobSpec jobSpec) { ((MutableJobCatalog)_fallback).put(jobSpec); }
@Override public Void apply(JobExecutionStateListener input) { input.onStatusChange(this.state, this.previousStatus, this.newStatus); return null; } }
@Override public Void apply(JobExecutionStateListener input) { input.onMetadataChange(state, key, oldValue, newValue); return null; }
/** {@inheritDoc} */ @Override public Properties getConfigAsProperties() { return _instanceConf.getConfigAsProperties(); }
@Override public Void apply(JobCatalogListener listener) { listener.onUpdateJob(_updatedJob); return null; } }
@Override public Integer getValue() { switch (parent.state()) { case RUNNING: return 1; case FAILED: return -1; default: return 0; } } });