@Override public AnalysisJobMetrics getAnalysisJobMetrics() { return _publishers.getAnalysisJobMetrics(); }
private RowProcessingPublisher getRowProcessingPublisher(final RowProcessingPublishers publishers) { final RowProcessingStream[] streams = publishers.getStreams(); if (streams.length != 1) { throw new UnsupportedOperationException("Jobs with multiple source tables are not distributable"); } final RowProcessingStream stream = streams[0]; return publishers.getRowProcessingPublisher(stream); }
private void registerJob(final AnalysisJob job) { final SourceColumnFinder sourceColumnFinder = new SourceColumnFinder(); sourceColumnFinder.addSources(job); for (final ComponentJob componentJob : getAllComponents(job)) { registerRowProcessingPublishers(sourceColumnFinder, job, componentJob); } }
/** * * @param table * @return * * @deprecated use {@link #getRowProcessingPublisher(RowProcessingStream)} * instead */ @Deprecated public RowProcessingPublisher getRowProcessingPublisher(final Table table) { final RowProcessingStream stream = getStream(table); return getRowProcessingPublisher(stream); }
new RowProcessingPublishers(analysisJob, analysisListener, errorAwareAnalysisListener, taskRunner, lifeCycleHelper); if (rowConsumeConfiguration.table != null) { @SuppressWarnings("deprecation") final RowProcessingPublisher tablePublisher = rowProcessingPublishers.getRowProcessingPublisher(rowConsumeConfiguration.table); if (tablePublisher == null) { throw new IllegalArgumentException( final Collection<RowProcessingPublisher> publishers = rowProcessingPublishers.getRowProcessingPublishers(); publisher = publishers.iterator().next(); for (final RowProcessingPublisher thisPublisher : publishers) {
(SourceTableRowProcessingPublisher) rowPublisher; final Column[] physicalColumns = getPhysicalColumns(sourceColumnFinder, componentJob); final Column[] relevantColumns = MetaModelHelper.getTableColumns(dataStream.getTable(), physicalColumns); getLocalInputColumns(sourceColumnFinder, dataStream.getTable(), componentJob.getInput()); final ConsumerCreation consumerCreation = getOrCreateConsumer(rowPublisher, componentJob, localInputColumns); final RowProcessingConsumer consumer = consumerCreation._consumer; final OutputDataStreamJob[] outputDataStreamJobs = componentJob.getOutputDataStreamJobs(); for (final OutputDataStreamJob outputDataStreamJob : outputDataStreamJobs) { registerOutputDataStream(rowPublisher, consumer, outputDataStreamJob);
new RowProcessingPublishers(_job, _analysisListener, _errorAware, _taskRunner, rowProcessingLifeCycleHelper); final AnalysisJobMetrics analysisJobMetrics = publishers.getAnalysisJobMetrics();
private void registerOutputDataStream(final RowProcessingPublisher parentPublisher, final RowProcessingConsumer publishingConsumer, final OutputDataStreamJob outputDataStreamJob) { final RowProcessingStream dataStream = RowProcessingStream.ofOutputDataStream(outputDataStreamJob); // first initialize the nested job like any other set of components registerJob(outputDataStreamJob.getJob(), dataStream, publishingConsumer); // then we wire the publisher for this output data stream to a // OutputRowCollector which will get injected via the // HasOutputDataStreams interface. final RowProcessingPublisher publisherForOutputDataStream = getRowProcessingPublisher(dataStream); publishingConsumer.registerOutputDataStream(outputDataStreamJob, publisherForOutputDataStream); }
/** * * @return * * @deprecated use {@link #getStreams()} instead */ @Deprecated public Table[] getTables() { final RowProcessingStream[] streams = getStreams(); final Table[] tables = new Table[streams.length]; for (int i = 0; i < tables.length; i++) { tables[i] = streams[i].getTable(); } return tables; }
protected final TaskRunnable createCloseTask(final RowProcessingConsumer consumer, final TaskListener closeTaskListener) { final LifeCycleHelper lifeCycleHelper = _publishers.getConsumerSpecificLifeCycleHelper(consumer); final CloseTaskListener taskListener = new CloseTaskListener(lifeCycleHelper, this, consumer, _success, closeTaskListener, _publishers.getAnalysisListener(), _stream.getAnalysisJob()); return new TaskRunnable(null, taskListener); }
private RowProcessingPublishers getRowProcessingPublishers(final AnalysisJob job, final LifeCycleHelper lifeCycleHelper) { final SingleThreadedTaskRunner taskRunner = new SingleThreadedTaskRunner(); final ErrorAwareAnalysisListener errorAwareAnalysisListener = new ErrorAwareAnalysisListener(); return new RowProcessingPublishers(job, errorAwareAnalysisListener, errorAwareAnalysisListener, taskRunner, lifeCycleHelper); }
public LifeCycleHelper getConsumerSpecificLifeCycleHelper(final RowProcessingConsumer consumer) { final LifeCycleHelper outerLifeCycleHelper = getLifeCycleHelper(); final boolean includeNonDistributedTasks = outerLifeCycleHelper.isIncludeNonDistributedTasks(); final InjectionManager outerInjectionManager = outerLifeCycleHelper.getInjectionManager(); final ContextAwareInjectionManager injectionManager = new ContextAwareInjectionManager(outerInjectionManager, consumer.getAnalysisJob(), consumer.getComponentJob(), getAnalysisListener()); return new LifeCycleHelper(injectionManager, includeNonDistributedTasks); }
@Override public final AnalysisListener getAnalysisListener() { return getPublishers().getAnalysisListener(); }
protected final TaskRunnable createInitTask(final RowProcessingConsumer consumer, final TaskListener listener) { final LifeCycleHelper lifeCycleHelper = _publishers.getConsumerSpecificLifeCycleHelper(consumer); final InitializeTask task = new InitializeTask(lifeCycleHelper, this, consumer); return new TaskRunnable(task, listener); }
@Override public ErrorAware getErrorAware() { return _publishers.getErrorAware(); }
@Override public RowProcessingMetrics getRowProcessingMetrics(final Table table) { final RowProcessingStream stream = _publishers.getStream(table); final RowProcessingPublisher publisher = _publishers.getRowProcessingPublisher(stream); if (publisher == null) { return null; } return publisher.getRowProcessingMetrics(); }
protected final Task createCollectResultTask(final RowProcessingConsumer consumer, final Queue<JobAndResult> resultQueue) { final Object component = consumer.getComponent(); if (component instanceof HasAnalyzerResult) { final HasAnalyzerResult<?> hasAnalyzerResult = (HasAnalyzerResult<?>) component; final AnalysisListener analysisListener = _publishers.getAnalysisListener(); return new CollectResultsTask(hasAnalyzerResult, _stream.getAnalysisJob(), consumer.getComponentJob(), resultQueue, analysisListener); } return null; }
private static void executeInternal(final RowProcessingConsumer consumer, final RowProcessingPublisher publisher, final LifeCycleHelper lifeCycleHelper) { // we synchronize to avoid a race condition where initialization // is on-going in one stream and therefore skipped in the other synchronized (consumer) { final int publisherCount = consumer.onPublisherInitialized(publisher); if (publisherCount == 1) { final ComponentConfiguration configuration = consumer.getComponentJob().getConfiguration(); final ComponentDescriptor<?> descriptor = consumer.getComponentJob().getDescriptor(); final Object component = consumer.getComponent(); lifeCycleHelper.assignConfiguredProperties(descriptor, component, configuration); lifeCycleHelper.assignProvidedProperties(descriptor, component); lifeCycleHelper.validate(descriptor, component); final Collection<ActiveOutputDataStream> activeOutputDataStreams = consumer.getActiveOutputDataStreams(); for (final ActiveOutputDataStream activeOutputDataStream : activeOutputDataStreams) { activeOutputDataStream.initialize(); final RowProcessingPublisher outputDataStreamPublisher = activeOutputDataStream.getPublisher(); for (final RowProcessingConsumer outputDataStreamConsumer : outputDataStreamPublisher .getConsumers()) { final LifeCycleHelper outputDataStreamLifeCycleHelper = outputDataStreamPublisher.getPublishers().getConsumerSpecificLifeCycleHelper(consumer); executeInternal(outputDataStreamConsumer, outputDataStreamPublisher, outputDataStreamLifeCycleHelper); } } lifeCycleHelper.initialize(descriptor, component); } } }
private void registerJob(final AnalysisJob job, final RowProcessingStream dataStream, final RowProcessingConsumer parentConsumer) { final SourceColumnFinder sourceColumnFinder = new SourceColumnFinder(); sourceColumnFinder.addSources(job); for (final ComponentJob componentJob : getAllComponents(job)) { registerRowProcessingPublishers(sourceColumnFinder, job, dataStream, componentJob, parentConsumer); } }
final AnalysisJobMetrics analysisJobMetrics = publishers.getAnalysisJobMetrics(); _analysisListener.jobBegin(job, analysisJobMetrics);