private void cleanup() { logger.debug("cleanup()"); final int publishersLeft = _consumer.onPublisherClosed(_publisher); if (publishersLeft == 0) { final Object component = _consumer.getComponent(); final ComponentDescriptor<?> descriptor = _consumer.getComponentJob().getDescriptor(); // close can occur AFTER completion _lifeCycleHelper.close(descriptor, component, _success.get()); _consumer.getActiveOutputDataStreams().forEach(ActiveOutputDataStream::close); } }
@Override public ComponentJob[] getResultProducers() { final List<ComponentJob> resultProducers = new ArrayList<>(); for (final RowProcessingConsumer consumer : _publisher.getConsumers()) { if (consumer.isResultProducer()) { resultProducers.add(consumer.getComponentJob()); } } return resultProducers.toArray(new ComponentJob[resultProducers.size()]); }
public ConsumeRowResult consume() { final RowProcessingConsumer consumer = _consumers.get(_consumerIndex); final boolean process = consumer.satisfiedForConsume(_outcomes, _row); if (process) { if (consumer.isConcurrent()) { consumer.consume(_row, 1, _outcomes, this); } else { synchronized (consumer) { consumer.consume(_row, 1, _outcomes, this); } } } else { // jump to the next step processNext(_row, 1, _outcomes); } return new ConsumeRowResult(_resultRecords, _resultOutcomes); }
@Override protected boolean isReadyForRowProcessing() { return _parentConsumer.isAllPublishersInitialized() && _parentConsumer.isAllPublishersClosed(); }
private List<Tuple2<String, NamedAnalyzerResult>> getAnalyzerResults( final Collection<RowProcessingConsumer> rowProcessingConsumers) { final List<Tuple2<String, NamedAnalyzerResult>> analyzerResults = new ArrayList<>(); for (final RowProcessingConsumer consumer : rowProcessingConsumers) { if (consumer.isResultProducer()) { final HasAnalyzerResult<?> resultProducer = (HasAnalyzerResult<?>) consumer.getComponent(); final AnalyzerResult analyzerResult = resultProducer.getResult(); final String key = _sparkJobContext.getComponentKey(consumer.getComponentJob()); final NamedAnalyzerResult namedAnalyzerResult = new NamedAnalyzerResult(key, analyzerResult); final Tuple2<String, NamedAnalyzerResult> tuple = new Tuple2<>(key, namedAnalyzerResult); analyzerResults.add(tuple); } for (final ActiveOutputDataStream activeOutputDataStream : consumer.getActiveOutputDataStreams()) { final List<RowProcessingConsumer> outputDataStreamConsumers = activeOutputDataStream.getPublisher().getConsumers(); final List<Tuple2<String, NamedAnalyzerResult>> outputDataStreamsAnalyzerResults = getAnalyzerResults(outputDataStreamConsumers); analyzerResults.addAll(outputDataStreamsAnalyzerResults); } } return analyzerResults; } }
private static void executeInternal(final RowProcessingConsumer consumer, final RowProcessingPublisher publisher, final LifeCycleHelper lifeCycleHelper) { // we synchronize to avoid a race condition where initialization // is on-going in one stream and therefore skipped in the other synchronized (consumer) { final int publisherCount = consumer.onPublisherInitialized(publisher); if (publisherCount == 1) { final ComponentConfiguration configuration = consumer.getComponentJob().getConfiguration(); final ComponentDescriptor<?> descriptor = consumer.getComponentJob().getDescriptor(); final Object component = consumer.getComponent(); lifeCycleHelper.assignConfiguredProperties(descriptor, component, configuration); lifeCycleHelper.assignProvidedProperties(descriptor, component); lifeCycleHelper.validate(descriptor, component); final Collection<ActiveOutputDataStream> activeOutputDataStreams = consumer.getActiveOutputDataStreams(); for (final ActiveOutputDataStream activeOutputDataStream : activeOutputDataStreams) { activeOutputDataStream.initialize(); final RowProcessingPublisher outputDataStreamPublisher = activeOutputDataStream.getPublisher(); for (final RowProcessingConsumer outputDataStreamConsumer : outputDataStreamPublisher .getConsumers()) { final LifeCycleHelper outputDataStreamLifeCycleHelper = outputDataStreamPublisher.getPublishers().getConsumerSpecificLifeCycleHelper(consumer); executeInternal(outputDataStreamConsumer, outputDataStreamPublisher, outputDataStreamLifeCycleHelper); } } lifeCycleHelper.initialize(descriptor, component); } } }
accepted = consumer.satisfiedForFlowOrdering(availableOutcomes); final InputColumn<?>[] requiredInput = consumer.getRequiredInput(); if (requiredInput != null) { for (final InputColumn<?> inputColumn : requiredInput) { changed = true; final ComponentJob componentJob = consumer.getComponentJob(); final InputColumn<?>[] requiredInput = consumer.getRequiredInput(); for (final InputColumn<?> inputColumn : requiredInput) { if (inputColumn instanceof ExpressionBasedInputColumn) { if (consumer.getComponent() instanceof MultiStreamComponent) { orderedConsumers.add(consumer); it.remove();
protected final Task createCollectResultTask(final RowProcessingConsumer consumer, final Queue<JobAndResult> resultQueue) { final Object component = consumer.getComponent(); if (component instanceof HasAnalyzerResult) { final HasAnalyzerResult<?> hasAnalyzerResult = (HasAnalyzerResult<?>) component; final AnalysisListener analysisListener = _publishers.getAnalysisListener(); return new CollectResultsTask(hasAnalyzerResult, _stream.getAnalysisJob(), consumer.getComponentJob(), resultQueue, analysisListener); } return null; }
public LifeCycleHelper getConsumerSpecificLifeCycleHelper(final RowProcessingConsumer consumer) { final LifeCycleHelper outerLifeCycleHelper = getLifeCycleHelper(); final boolean includeNonDistributedTasks = outerLifeCycleHelper.isIncludeNonDistributedTasks(); final InjectionManager outerInjectionManager = outerLifeCycleHelper.getInjectionManager(); final ContextAwareInjectionManager injectionManager = new ContextAwareInjectionManager(outerInjectionManager, consumer.getAnalysisJob(), consumer.getComponentJob(), getAnalysisListener()); return new LifeCycleHelper(injectionManager, includeNonDistributedTasks); }
@Override public final RowProcessingConsumer getConsumer(final ComponentJob componentJob) { for (final RowProcessingConsumer consumer : _consumers) { if (componentJob.equals(consumer.getComponentJob())) { return consumer; } } return null; }
private List<RowProcessingConsumer> removeAnalyzers(final List<RowProcessingConsumer> consumers) { final List<RowProcessingConsumer> result = new ArrayList<>(); for (final RowProcessingConsumer rowProcessingConsumer : consumers) { final Object component = rowProcessingConsumer.getComponent(); if (!(component instanceof Analyzer<?>)) { result.add(rowProcessingConsumer); } } return result; } }
@Override protected boolean processRowsInternal(final AnalysisListener listener, final RowProcessingMetrics rowProcessingMetrics) { final Collection<ActiveOutputDataStream> activeOutputDataStreams = _parentConsumer.getActiveOutputDataStreams(); for (final ActiveOutputDataStream activeOutputDataStream : activeOutputDataStreams) { try { activeOutputDataStream.await(); } catch (final InterruptedException e) { logger.error("Unexpected error awaiting output data stream", e); listener.errorUnknown(getAnalysisJob(), e); return false; } } return true; }
/** * Gets the output columns produced by all the consumers of this * {@link ConsumeRowHandler}. * * @return */ public List<InputColumn<?>> getOutputColumns() { final List<InputColumn<?>> result = new ArrayList<>(); for (final RowProcessingConsumer consumer : _consumers) { final InputColumn<?>[] outputColumns = consumer.getOutputColumns(); for (final InputColumn<?> outputColumn : outputColumns) { result.add(outputColumn); } } return result; }
lifeCycleHelper.close(consumer.getComponentJob().getDescriptor(), consumer.getComponent(), true);
@Override public final ConsumeRowHandler createConsumeRowHandler() { final RowProcessingQueryOptimizer queryOptimizer = getQueryOptimizer(); final Query finalQuery = queryOptimizer.getOptimizedQuery(); final RowIdGenerator idGenerator; if (finalQuery.getFirstRow() == null) { idGenerator = new SimpleRowIdGenerator(); } else { idGenerator = new SimpleRowIdGenerator(finalQuery.getFirstRow()); } final RowProcessingPublishers publishers = getPublishers(); final AnalysisListener analysisListener = publishers.getAnalysisListener(); for (final RowProcessingConsumer consumer : getConsumers()) { final ComponentJob componentJob = consumer.getComponentJob(); final ComponentMetrics metrics = new AnalysisJobMetricsImpl(consumer.getAnalysisJob(), publishers).getComponentMetrics(componentJob); analysisListener.componentBegin(getStream().getAnalysisJob(), componentJob, metrics); if (consumer instanceof TransformerConsumer) { ((TransformerConsumer) consumer).setRowIdGenerator(idGenerator); } } final List<RowProcessingConsumer> consumers = queryOptimizer.getOptimizedConsumers(); final Collection<? extends FilterOutcome> availableOutcomes = queryOptimizer.getOptimizedAvailableOutcomes(); return new ConsumeRowHandler(consumers, availableOutcomes); }
final ComponentJob componentJob = nextConsumer.getComponentJob(); if (componentJob instanceof HasComponentRequirement) { final ComponentRequirement componentRequirement = componentJob.getComponentRequirement();