@Override public final RowProcessingConsumer getConsumer(final ComponentJob componentJob) { for (final RowProcessingConsumer consumer : _consumers) { if (componentJob.equals(consumer.getComponentJob())) { return consumer; } } return null; }
@Override public ComponentJob[] getResultProducers() { final List<ComponentJob> resultProducers = new ArrayList<>(); for (final RowProcessingConsumer consumer : _publisher.getConsumers()) { if (consumer.isResultProducer()) { resultProducers.add(consumer.getComponentJob()); } } return resultProducers.toArray(new ComponentJob[resultProducers.size()]); }
private void cleanup() { logger.debug("cleanup()"); final int publishersLeft = _consumer.onPublisherClosed(_publisher); if (publishersLeft == 0) { final Object component = _consumer.getComponent(); final ComponentDescriptor<?> descriptor = _consumer.getComponentJob().getDescriptor(); // close can occur AFTER completion _lifeCycleHelper.close(descriptor, component, _success.get()); _consumer.getActiveOutputDataStreams().forEach(ActiveOutputDataStream::close); } }
protected final Task createCollectResultTask(final RowProcessingConsumer consumer, final Queue<JobAndResult> resultQueue) { final Object component = consumer.getComponent(); if (component instanceof HasAnalyzerResult) { final HasAnalyzerResult<?> hasAnalyzerResult = (HasAnalyzerResult<?>) component; final AnalysisListener analysisListener = _publishers.getAnalysisListener(); return new CollectResultsTask(hasAnalyzerResult, _stream.getAnalysisJob(), consumer.getComponentJob(), resultQueue, analysisListener); } return null; }
changed = true; final ComponentJob componentJob = consumer.getComponentJob();
final ComponentJob componentJob = nextConsumer.getComponentJob(); if (componentJob instanceof HasComponentRequirement) { final ComponentRequirement componentRequirement = componentJob.getComponentRequirement();
private List<Tuple2<String, NamedAnalyzerResult>> getAnalyzerResults( final Collection<RowProcessingConsumer> rowProcessingConsumers) { final List<Tuple2<String, NamedAnalyzerResult>> analyzerResults = new ArrayList<>(); for (final RowProcessingConsumer consumer : rowProcessingConsumers) { if (consumer.isResultProducer()) { final HasAnalyzerResult<?> resultProducer = (HasAnalyzerResult<?>) consumer.getComponent(); final AnalyzerResult analyzerResult = resultProducer.getResult(); final String key = _sparkJobContext.getComponentKey(consumer.getComponentJob()); final NamedAnalyzerResult namedAnalyzerResult = new NamedAnalyzerResult(key, analyzerResult); final Tuple2<String, NamedAnalyzerResult> tuple = new Tuple2<>(key, namedAnalyzerResult); analyzerResults.add(tuple); } for (final ActiveOutputDataStream activeOutputDataStream : consumer.getActiveOutputDataStreams()) { final List<RowProcessingConsumer> outputDataStreamConsumers = activeOutputDataStream.getPublisher().getConsumers(); final List<Tuple2<String, NamedAnalyzerResult>> outputDataStreamsAnalyzerResults = getAnalyzerResults(outputDataStreamConsumers); analyzerResults.addAll(outputDataStreamsAnalyzerResults); } } return analyzerResults; } }
public LifeCycleHelper getConsumerSpecificLifeCycleHelper(final RowProcessingConsumer consumer) { final LifeCycleHelper outerLifeCycleHelper = getLifeCycleHelper(); final boolean includeNonDistributedTasks = outerLifeCycleHelper.isIncludeNonDistributedTasks(); final InjectionManager outerInjectionManager = outerLifeCycleHelper.getInjectionManager(); final ContextAwareInjectionManager injectionManager = new ContextAwareInjectionManager(outerInjectionManager, consumer.getAnalysisJob(), consumer.getComponentJob(), getAnalysisListener()); return new LifeCycleHelper(injectionManager, includeNonDistributedTasks); }
lifeCycleHelper.close(consumer.getComponentJob().getDescriptor(), consumer.getComponent(), true);
private static void executeInternal(final RowProcessingConsumer consumer, final RowProcessingPublisher publisher, final LifeCycleHelper lifeCycleHelper) { // we synchronize to avoid a race condition where initialization // is on-going in one stream and therefore skipped in the other synchronized (consumer) { final int publisherCount = consumer.onPublisherInitialized(publisher); if (publisherCount == 1) { final ComponentConfiguration configuration = consumer.getComponentJob().getConfiguration(); final ComponentDescriptor<?> descriptor = consumer.getComponentJob().getDescriptor(); final Object component = consumer.getComponent(); lifeCycleHelper.assignConfiguredProperties(descriptor, component, configuration); lifeCycleHelper.assignProvidedProperties(descriptor, component); lifeCycleHelper.validate(descriptor, component); final Collection<ActiveOutputDataStream> activeOutputDataStreams = consumer.getActiveOutputDataStreams(); for (final ActiveOutputDataStream activeOutputDataStream : activeOutputDataStreams) { activeOutputDataStream.initialize(); final RowProcessingPublisher outputDataStreamPublisher = activeOutputDataStream.getPublisher(); for (final RowProcessingConsumer outputDataStreamConsumer : outputDataStreamPublisher .getConsumers()) { final LifeCycleHelper outputDataStreamLifeCycleHelper = outputDataStreamPublisher.getPublishers().getConsumerSpecificLifeCycleHelper(consumer); executeInternal(outputDataStreamConsumer, outputDataStreamPublisher, outputDataStreamLifeCycleHelper); } } lifeCycleHelper.initialize(descriptor, component); } } }
@Override public final ConsumeRowHandler createConsumeRowHandler() { final RowProcessingQueryOptimizer queryOptimizer = getQueryOptimizer(); final Query finalQuery = queryOptimizer.getOptimizedQuery(); final RowIdGenerator idGenerator; if (finalQuery.getFirstRow() == null) { idGenerator = new SimpleRowIdGenerator(); } else { idGenerator = new SimpleRowIdGenerator(finalQuery.getFirstRow()); } final RowProcessingPublishers publishers = getPublishers(); final AnalysisListener analysisListener = publishers.getAnalysisListener(); for (final RowProcessingConsumer consumer : getConsumers()) { final ComponentJob componentJob = consumer.getComponentJob(); final ComponentMetrics metrics = new AnalysisJobMetricsImpl(consumer.getAnalysisJob(), publishers).getComponentMetrics(componentJob); analysisListener.componentBegin(getStream().getAnalysisJob(), componentJob, metrics); if (consumer instanceof TransformerConsumer) { ((TransformerConsumer) consumer).setRowIdGenerator(idGenerator); } } final List<RowProcessingConsumer> consumers = queryOptimizer.getOptimizedConsumers(); final Collection<? extends FilterOutcome> availableOutcomes = queryOptimizer.getOptimizedAvailableOutcomes(); return new ConsumeRowHandler(consumers, availableOutcomes); }