/** * Closes consumers of this {@link SourceTableRowProcessingPublisher}. * Usually this will be done automatically when * {@link #runRowProcessing(Queue, TaskListener)} is invoked. */ @Override public final void closeConsumers() { final TaskRunner taskRunner = getTaskRunner(); final List<RowProcessingConsumer> configurableConsumers = getConsumers(); for (final RowProcessingConsumer consumer : configurableConsumers) { final TaskRunnable task = createCloseTask(consumer, null); taskRunner.run(task); } } }
@Override public final boolean runRowProcessing(final Queue<JobAndResult> resultQueue, final TaskListener finishedTaskListener) { if (!isReadyForRowProcessing()) { return false; } final List<TaskRunnable> postProcessingTasks = createPostProcessingTasks(resultQueue, finishedTaskListener); return runRowProcessingInternal(postProcessingTasks); }
@Override public final void processRows(final RowProcessingMetrics rowProcessingMetrics) { final AnalysisListener analysisListener = getAnalysisListener(); final boolean success = processRowsInternal(analysisListener, rowProcessingMetrics); if (!success) { _success.set(false); return; } analysisListener.rowProcessingSuccess(getAnalysisJob(), rowProcessingMetrics); }
private List<TaskRunnable> createPostProcessingTasks(final Queue<JobAndResult> resultQueue, final TaskListener finishedTaskListener) { final List<RowProcessingConsumer> configurableConsumers = getConsumers(); final int numConsumers = configurableConsumers.size(); // add tasks for closing components final JoinTaskListener closeTaskListener = new JoinTaskListener(numConsumers, finishedTaskListener); final List<TaskRunnable> closeTasks = new ArrayList<>(); for (final RowProcessingConsumer consumer : configurableConsumers) { closeTasks.add(createCloseTask(consumer, closeTaskListener)); } final TaskListener getResultCompletionListener = new ForkTaskListener("collect results (" + getStream() + ")", getTaskRunner(), closeTasks); // add tasks for collecting results final TaskListener getResultTaskListener = new JoinTaskListener(numConsumers, getResultCompletionListener); final List<TaskRunnable> getResultTasks = new ArrayList<>(); for (final RowProcessingConsumer consumer : configurableConsumers) { final Task collectResultTask = createCollectResultTask(consumer, resultQueue); if (collectResultTask == null) { getResultTasks.add(new TaskRunnable(null, getResultTaskListener)); } else { getResultTasks.add(new TaskRunnable(collectResultTask, getResultTaskListener)); } } return getResultTasks; }
@Override public final ConsumeRowHandler createConsumeRowHandler() { final RowProcessingQueryOptimizer queryOptimizer = getQueryOptimizer(); final Query finalQuery = queryOptimizer.getOptimizedQuery(); final RowIdGenerator idGenerator; if (finalQuery.getFirstRow() == null) { idGenerator = new SimpleRowIdGenerator(); } else { idGenerator = new SimpleRowIdGenerator(finalQuery.getFirstRow()); } final RowProcessingPublishers publishers = getPublishers(); final AnalysisListener analysisListener = publishers.getAnalysisListener(); for (final RowProcessingConsumer consumer : getConsumers()) { final ComponentJob componentJob = consumer.getComponentJob(); final ComponentMetrics metrics = new AnalysisJobMetricsImpl(consumer.getAnalysisJob(), publishers).getComponentMetrics(componentJob); analysisListener.componentBegin(getStream().getAnalysisJob(), componentJob, metrics); if (consumer instanceof TransformerConsumer) { ((TransformerConsumer) consumer).setRowIdGenerator(idGenerator); } } final List<RowProcessingConsumer> consumers = queryOptimizer.getOptimizedConsumers(); final Collection<? extends FilterOutcome> availableOutcomes = queryOptimizer.getOptimizedAvailableOutcomes(); return new ConsumeRowHandler(consumers, availableOutcomes); }
/** * Initializes consumers of this {@link SourceTableRowProcessingPublisher}. * * This method will not initialize consumers containing * {@link MultiStreamComponent}s. Ensure that * {@link #initializeMultiStreamConsumers(Set)} is also invoked. * * Once consumers are initialized, row processing can begin, expected rows * can be calculated and more. * * @param finishedListener */ @Override public final void initializeConsumers(final TaskListener finishedListener) { final TaskRunner taskRunner = getTaskRunner(); final List<RowProcessingConsumer> configurableConsumers = getConsumers(); final int numConfigurableConsumers = configurableConsumers.size(); final JoinTaskListener initFinishedListener = new JoinTaskListener(numConfigurableConsumers, finishedListener); for (final RowProcessingConsumer consumer : configurableConsumers) { final TaskRunnable task = createInitTask(consumer, initFinishedListener); taskRunner.run(task); } }
@Override public final String toString() { return getClass().getSimpleName() + "[stream=" + getStream() + ", consumers=" + _consumers.size() + "]"; }
protected final TaskRunner getTaskRunner() { return getPublishers().getTaskRunner(); }
@Override public final Query getQuery() { return getQueryOptimizer().getOptimizedQuery(); }
protected final List<RowProcessingConsumer> getConsumersSorted() { final List<RowProcessingConsumer> consumers = getConsumers(); final RowProcessingConsumerSorter sorter = new RowProcessingConsumerSorter(consumers); return sorter.createProcessOrderedConsumerList(); }
@Override public final AnalysisJob getAnalysisJob() { return getStream().getAnalysisJob(); }
@Override public final AnalysisListener getAnalysisListener() { return getPublishers().getAnalysisListener(); }