@Override public AnalyzerJob[] getAnalyzerJobs() { final List<AnalyzerJob> analyzerJobs = new ArrayList<>(); for (final RowProcessingConsumer consumer : _publisher.getConsumers()) { if (consumer instanceof AnalyzerConsumer) { final AnalyzerJob analyzerJob = ((AnalyzerConsumer) consumer).getComponentJob(); analyzerJobs.add(analyzerJob); } } return analyzerJobs.toArray(new AnalyzerJob[analyzerJobs.size()]); }
@Override public ComponentJob[] getResultProducers() { final List<ComponentJob> resultProducers = new ArrayList<>(); for (final RowProcessingConsumer consumer : _publisher.getConsumers()) { if (consumer.isResultProducer()) { resultProducers.add(consumer.getComponentJob()); } } return resultProducers.toArray(new ComponentJob[resultProducers.size()]); }
private List<Tuple2<String, NamedAnalyzerResult>> getAnalyzerResults( final Collection<RowProcessingConsumer> rowProcessingConsumers) { final List<Tuple2<String, NamedAnalyzerResult>> analyzerResults = new ArrayList<>(); for (final RowProcessingConsumer consumer : rowProcessingConsumers) { if (consumer.isResultProducer()) { final HasAnalyzerResult<?> resultProducer = (HasAnalyzerResult<?>) consumer.getComponent(); final AnalyzerResult analyzerResult = resultProducer.getResult(); final String key = _sparkJobContext.getComponentKey(consumer.getComponentJob()); final NamedAnalyzerResult namedAnalyzerResult = new NamedAnalyzerResult(key, analyzerResult); final Tuple2<String, NamedAnalyzerResult> tuple = new Tuple2<>(key, namedAnalyzerResult); analyzerResults.add(tuple); } for (final ActiveOutputDataStream activeOutputDataStream : consumer.getActiveOutputDataStreams()) { final List<RowProcessingConsumer> outputDataStreamConsumers = activeOutputDataStream.getPublisher().getConsumers(); final List<Tuple2<String, NamedAnalyzerResult>> outputDataStreamsAnalyzerResults = getAnalyzerResults(outputDataStreamConsumers); analyzerResults.addAll(outputDataStreamsAnalyzerResults); } } return analyzerResults; } }
List<RowProcessingConsumer> consumers = publisher.getConsumers(); if (!rowConsumeConfiguration.includeAnalyzers) { consumers = removeAnalyzers(consumers);
private static void executeInternal(final RowProcessingConsumer consumer, final RowProcessingPublisher publisher, final LifeCycleHelper lifeCycleHelper) { // we synchronize to avoid a race condition where initialization // is on-going in one stream and therefore skipped in the other synchronized (consumer) { final int publisherCount = consumer.onPublisherInitialized(publisher); if (publisherCount == 1) { final ComponentConfiguration configuration = consumer.getComponentJob().getConfiguration(); final ComponentDescriptor<?> descriptor = consumer.getComponentJob().getDescriptor(); final Object component = consumer.getComponent(); lifeCycleHelper.assignConfiguredProperties(descriptor, component, configuration); lifeCycleHelper.assignProvidedProperties(descriptor, component); lifeCycleHelper.validate(descriptor, component); final Collection<ActiveOutputDataStream> activeOutputDataStreams = consumer.getActiveOutputDataStreams(); for (final ActiveOutputDataStream activeOutputDataStream : activeOutputDataStreams) { activeOutputDataStream.initialize(); final RowProcessingPublisher outputDataStreamPublisher = activeOutputDataStream.getPublisher(); for (final RowProcessingConsumer outputDataStreamConsumer : outputDataStreamPublisher .getConsumers()) { final LifeCycleHelper outputDataStreamLifeCycleHelper = outputDataStreamPublisher.getPublishers().getConsumerSpecificLifeCycleHelper(consumer); executeInternal(outputDataStreamConsumer, outputDataStreamPublisher, outputDataStreamLifeCycleHelper); } } lifeCycleHelper.initialize(descriptor, component); } } }