public static RowProcessingStream ofSourceTable(final AnalysisJob analysisJob, final Table table) { return new RowProcessingStream(analysisJob, table, true); }
@Override public final AnalysisJob getAnalysisJob() { return getStream().getAnalysisJob(); }
public RowProcessingStream getStream(final Table table) { final Set<RowProcessingStream> dataStreams = _rowProcessingPublishers.keySet(); for (final RowProcessingStream stream : dataStreams) { // first try with object equality because tables may be equal in // some corner cases if (stream.getTable() == table) { return stream; } } for (final RowProcessingStream stream : dataStreams) { if (table.equals(stream.getTable())) { return stream; } } return null; }
private void registerRowProcessingPublishers(final SourceColumnFinder sourceColumnFinder, final AnalysisJob job, final ComponentJob componentJob) { final Column[] physicalColumns = getPhysicalColumns(sourceColumnFinder, componentJob); final Table[] tables = getTables(sourceColumnFinder, componentJob, physicalColumns); for (final Table table : tables) { final RowProcessingStream dataStream = RowProcessingStream.ofSourceTable(job, table); registerRowProcessingPublishers(sourceColumnFinder, job, dataStream, componentJob, null); } }
private void registerOutputDataStream(final RowProcessingPublisher parentPublisher, final RowProcessingConsumer publishingConsumer, final OutputDataStreamJob outputDataStreamJob) { final RowProcessingStream dataStream = RowProcessingStream.ofOutputDataStream(outputDataStreamJob); // first initialize the nested job like any other set of components registerJob(outputDataStreamJob.getJob(), dataStream, publishingConsumer); // then we wire the publisher for this output data stream to a // OutputRowCollector which will get injected via the // HasOutputDataStreams interface. final RowProcessingPublisher publisherForOutputDataStream = getRowProcessingPublisher(dataStream); publishingConsumer.registerOutputDataStream(outputDataStreamJob, publisherForOutputDataStream); }
for (final RowProcessingPublisher thisPublisher : publishers) { if (thisPublisher != publisher) { if (thisPublisher.getStream().isSourceTable()) { throw new IllegalArgumentException( "Job consumes multiple source tables, but ConsumeRowHandler can only handle a single "
public AbstractRowProcessingPublisher(final RowProcessingPublishers publishers, final RowProcessingStream stream) { if (publishers == null) { throw new IllegalArgumentException("RowProcessingPublishers cannot be null"); } if (stream == null) { throw new IllegalArgumentException("RowProcessingStream cannot be null"); } _publishers = publishers; _stream = stream; _sourceColumnFinder = new SourceColumnFinder(); _sourceColumnFinder.addSources(stream.getAnalysisJob()); _consumers = new ArrayList<>(); _success = new AtomicBoolean(true); }
@Override public String toString() { return getTable().getName(); } }
public static RowProcessingStream ofOutputDataStream(final OutputDataStreamJob outputDataStreamJob) { return new RowProcessingStream(outputDataStreamJob.getJob(), outputDataStreamJob.getOutputDataStream().getTable(), false); }
protected final Task createCollectResultTask(final RowProcessingConsumer consumer, final Queue<JobAndResult> resultQueue) { final Object component = consumer.getComponent(); if (component instanceof HasAnalyzerResult) { final HasAnalyzerResult<?> hasAnalyzerResult = (HasAnalyzerResult<?>) component; final AnalysisListener analysisListener = _publishers.getAnalysisListener(); return new CollectResultsTask(hasAnalyzerResult, _stream.getAnalysisJob(), consumer.getComponentJob(), resultQueue, analysisListener); } return null; }
@Override public Table getTable() { return _publisher.getStream().getTable(); }
protected final TaskRunnable createCloseTask(final RowProcessingConsumer consumer, final TaskListener closeTaskListener) { final LifeCycleHelper lifeCycleHelper = _publishers.getConsumerSpecificLifeCycleHelper(consumer); final CloseTaskListener taskListener = new CloseTaskListener(lifeCycleHelper, this, consumer, _success, closeTaskListener, _publishers.getAnalysisListener(), _stream.getAnalysisJob()); return new TaskRunnable(null, taskListener); }
private Table getTable() { return getStream().getTable(); }
/** * Constructor to use for creating a * {@link SourceTableRowProcessingPublisher} which feeds data from a source * datastore. * * @param publishers * @param stream */ public SourceTableRowProcessingPublisher(final RowProcessingPublishers publishers, final RowProcessingStream stream) { super(publishers, stream); _queryOptimizerRef = createQueryOptimizerRef(); final boolean aggressiveOptimizeSelectClause = SystemProperties.getBoolean(SystemProperties.QUERY_SELECTCLAUSE_OPTIMIZE, false); if (!aggressiveOptimizeSelectClause) { final Collection<InputColumn<?>> sourceColumns = stream.getAnalysisJob().getSourceColumns(); final List<Column> columns = new ArrayList<>(); for (final InputColumn<?> sourceColumn : sourceColumns) { final Column column = sourceColumn.getPhysicalColumn(); if (column != null && getTable().equals(column.getTable())) { columns.add(column); } } addPhysicalColumns(columns); } }
/** * * @return * * @deprecated use {@link #getStreams()} instead */ @Deprecated public Table[] getTables() { final RowProcessingStream[] streams = getStreams(); final Table[] tables = new Table[streams.length]; for (int i = 0; i < tables.length; i++) { tables[i] = streams[i].getTable(); } return tables; }
countQuery.getSelectClause().getItem(0).setFunctionApproximationAllowed(true); final Datastore datastore = _publisher.getStream().getAnalysisJob().getDatastore(); try (DatastoreConnection connection = datastore.openConnection()) { try (DataSet countDataSet = connection.getDataContext().executeQuery(countQuery)) {
@Override public final ConsumeRowHandler createConsumeRowHandler() { final RowProcessingQueryOptimizer queryOptimizer = getQueryOptimizer(); final Query finalQuery = queryOptimizer.getOptimizedQuery(); final RowIdGenerator idGenerator; if (finalQuery.getFirstRow() == null) { idGenerator = new SimpleRowIdGenerator(); } else { idGenerator = new SimpleRowIdGenerator(finalQuery.getFirstRow()); } final RowProcessingPublishers publishers = getPublishers(); final AnalysisListener analysisListener = publishers.getAnalysisListener(); for (final RowProcessingConsumer consumer : getConsumers()) { final ComponentJob componentJob = consumer.getComponentJob(); final ComponentMetrics metrics = new AnalysisJobMetricsImpl(consumer.getAnalysisJob(), publishers).getComponentMetrics(componentJob); analysisListener.componentBegin(getStream().getAnalysisJob(), componentJob, metrics); if (consumer instanceof TransformerConsumer) { ((TransformerConsumer) consumer).setRowIdGenerator(idGenerator); } } final List<RowProcessingConsumer> consumers = queryOptimizer.getOptimizedConsumers(); final Collection<? extends FilterOutcome> availableOutcomes = queryOptimizer.getOptimizedAvailableOutcomes(); return new ConsumeRowHandler(consumers, availableOutcomes); }
@Override protected RowProcessingQueryOptimizer getQueryOptimizer() { final Table table = getStream().getTable(); final Query q = new Query().from(table).select(table.getColumns()); return new NoopRowProcessingQueryOptimizer(q, getConsumersSorted()); } }