private List<RowProcessingConsumer> removeAnalyzers(final List<RowProcessingConsumer> consumers) { final List<RowProcessingConsumer> result = new ArrayList<>(); for (final RowProcessingConsumer rowProcessingConsumer : consumers) { final Object component = rowProcessingConsumer.getComponent(); if (!(component instanceof Analyzer<?>)) { result.add(rowProcessingConsumer); } } return result; } }
private void cleanup() { logger.debug("cleanup()"); final int publishersLeft = _consumer.onPublisherClosed(_publisher); if (publishersLeft == 0) { final Object component = _consumer.getComponent(); final ComponentDescriptor<?> descriptor = _consumer.getComponentJob().getDescriptor(); // close can occur AFTER completion _lifeCycleHelper.close(descriptor, component, _success.get()); _consumer.getActiveOutputDataStreams().forEach(ActiveOutputDataStream::close); } }
protected final Task createCollectResultTask(final RowProcessingConsumer consumer, final Queue<JobAndResult> resultQueue) { final Object component = consumer.getComponent(); if (component instanceof HasAnalyzerResult) { final HasAnalyzerResult<?> hasAnalyzerResult = (HasAnalyzerResult<?>) component; final AnalysisListener analysisListener = _publishers.getAnalysisListener(); return new CollectResultsTask(hasAnalyzerResult, _stream.getAnalysisJob(), consumer.getComponentJob(), resultQueue, analysisListener); } return null; }
private List<Tuple2<String, NamedAnalyzerResult>> getAnalyzerResults( final Collection<RowProcessingConsumer> rowProcessingConsumers) { final List<Tuple2<String, NamedAnalyzerResult>> analyzerResults = new ArrayList<>(); for (final RowProcessingConsumer consumer : rowProcessingConsumers) { if (consumer.isResultProducer()) { final HasAnalyzerResult<?> resultProducer = (HasAnalyzerResult<?>) consumer.getComponent(); final AnalyzerResult analyzerResult = resultProducer.getResult(); final String key = _sparkJobContext.getComponentKey(consumer.getComponentJob()); final NamedAnalyzerResult namedAnalyzerResult = new NamedAnalyzerResult(key, analyzerResult); final Tuple2<String, NamedAnalyzerResult> tuple = new Tuple2<>(key, namedAnalyzerResult); analyzerResults.add(tuple); } for (final ActiveOutputDataStream activeOutputDataStream : consumer.getActiveOutputDataStreams()) { final List<RowProcessingConsumer> outputDataStreamConsumers = activeOutputDataStream.getPublisher().getConsumers(); final List<Tuple2<String, NamedAnalyzerResult>> outputDataStreamsAnalyzerResults = getAnalyzerResults(outputDataStreamConsumers); analyzerResults.addAll(outputDataStreamsAnalyzerResults); } } return analyzerResults; } }
if (consumer.getComponent() instanceof MultiStreamComponent) { orderedConsumers.add(consumer); it.remove();
lifeCycleHelper.close(consumer.getComponentJob().getDescriptor(), consumer.getComponent(), true);
private static void executeInternal(final RowProcessingConsumer consumer, final RowProcessingPublisher publisher, final LifeCycleHelper lifeCycleHelper) { // we synchronize to avoid a race condition where initialization // is on-going in one stream and therefore skipped in the other synchronized (consumer) { final int publisherCount = consumer.onPublisherInitialized(publisher); if (publisherCount == 1) { final ComponentConfiguration configuration = consumer.getComponentJob().getConfiguration(); final ComponentDescriptor<?> descriptor = consumer.getComponentJob().getDescriptor(); final Object component = consumer.getComponent(); lifeCycleHelper.assignConfiguredProperties(descriptor, component, configuration); lifeCycleHelper.assignProvidedProperties(descriptor, component); lifeCycleHelper.validate(descriptor, component); final Collection<ActiveOutputDataStream> activeOutputDataStreams = consumer.getActiveOutputDataStreams(); for (final ActiveOutputDataStream activeOutputDataStream : activeOutputDataStreams) { activeOutputDataStream.initialize(); final RowProcessingPublisher outputDataStreamPublisher = activeOutputDataStream.getPublisher(); for (final RowProcessingConsumer outputDataStreamConsumer : outputDataStreamPublisher .getConsumers()) { final LifeCycleHelper outputDataStreamLifeCycleHelper = outputDataStreamPublisher.getPublishers().getConsumerSpecificLifeCycleHelper(consumer); executeInternal(outputDataStreamConsumer, outputDataStreamPublisher, outputDataStreamLifeCycleHelper); } } lifeCycleHelper.initialize(descriptor, component); } } }