/** * Gets the parallelism for this operator. * * @return The parallelism set for this operator. */ public int getParallelism() { return transformation.getParallelism(); }
/** * Creates a new {@code SplitTransformation} from the given input and {@code OutputSelector}. * * @param input The input {@code StreamTransformation} * @param outputSelector The output selector */ public SplitTransformation(StreamTransformation<T> input, OutputSelector<T> outputSelector) { super("Split", input.getOutputType(), input.getParallelism()); this.input = input; this.outputSelector = outputSelector; }
/** * Creates a new {@code PartitionTransformation} from the given input and * {@link StreamPartitioner}. * * @param input The input {@code StreamTransformation} * @param partitioner The {@code StreamPartitioner} */ public PartitionTransformation(StreamTransformation<T> input, StreamPartitioner<T> partitioner) { super("Partition", input.getOutputType(), input.getParallelism()); this.input = input; this.partitioner = partitioner; }
public SideOutputTransformation(StreamTransformation<?> input, final OutputTag<T> tag) { super("SideOutput", tag.getTypeInfo(), requireNonNull(input).getParallelism()); this.input = input; this.tag = requireNonNull(tag); }
/** * Creates a new {@code SelectionTransformation} from the given input that only selects * the streams with the selected names. * * @param input The input {@code StreamTransformation} * @param selectedNames The names from the upstream {@code SplitTransformation} that this * {@code SelectTransformation} selects. */ public SelectTransformation(StreamTransformation<T> input, List<String> selectedNames) { super("Select", input.getOutputType(), input.getParallelism()); this.input = input; this.selectedNames = selectedNames; }
/** * Adds a feedback edge. The parallelism of the {@code StreamTransformation} must match * the parallelism of the input {@code StreamTransformation} of the upstream * {@code StreamTransformation}. * * @param transform The new feedback {@code StreamTransformation}. */ public void addFeedbackEdge(StreamTransformation<F> transform) { if (transform.getParallelism() != this.getParallelism()) { throw new UnsupportedOperationException( "Parallelism of the feedback stream must match the parallelism of the original" + " stream. Parallelism of original stream: " + this.getParallelism() + "; parallelism of feedback stream: " + transform.getParallelism()); } feedbackEdges.add(transform); }
/** * Adds a feedback edge. The parallelism of the {@code StreamTransformation} must match * the parallelism of the input {@code StreamTransformation} of this * {@code FeedbackTransformation} * * @param transform The new feedback {@code StreamTransformation}. */ public void addFeedbackEdge(StreamTransformation<T> transform) { if (transform.getParallelism() != this.getParallelism()) { throw new UnsupportedOperationException( "Parallelism of the feedback stream must match the parallelism of the original" + " stream. Parallelism of original stream: " + this.getParallelism() + "; parallelism of feedback stream: " + transform.getParallelism() + ". Parallelism can be modified using DataStream#setParallelism() method"); } feedbackEdges.add(transform); }
/** * Creates a new {@code FeedbackTransformation} from the given input. * * @param input The input {@code StreamTransformation} * @param waitTime The wait time of the feedback operator. After the time expires * the operation will close and not receive any more feedback elements. */ public FeedbackTransformation(StreamTransformation<T> input, Long waitTime) { super("Feedback", input.getOutputType(), input.getParallelism()); this.input = input; this.waitTime = waitTime; this.feedbackEdges = Lists.newArrayList(); }
/** * Creates a new {@code UnionTransformation} from the given input {@code StreamTransformations}. * * <p>The input {@code StreamTransformations} must all have the same type. * * @param inputs The list of input {@code StreamTransformations} */ public UnionTransformation(List<StreamTransformation<T>> inputs) { super("Union", inputs.get(0).getOutputType(), inputs.get(0).getParallelism()); for (StreamTransformation<T> input: inputs) { if (!input.getOutputType().equals(getOutputType())) { throw new UnsupportedOperationException("Type mismatch in input " + input); } } this.inputs = Lists.newArrayList(inputs); }
/** * Extracts a timestamp from an element and assigns it as the internal timestamp of that element. * The internal timestamps are, for example, used to to event-time window operations. * * <p>If you know that the timestamps are strictly increasing you can use an * {@link AscendingTimestampExtractor}. Otherwise, * you should provide a {@link TimestampExtractor} that also implements * {@link TimestampExtractor#getCurrentWatermark()} to keep track of watermarks. * * @param extractor The TimestampExtractor that is called for each element of the DataStream. * * @deprecated Please use {@link #assignTimestampsAndWatermarks(AssignerWithPeriodicWatermarks)} * of {@link #assignTimestampsAndWatermarks(AssignerWithPunctuatedWatermarks)} * instead. * @see #assignTimestampsAndWatermarks(AssignerWithPeriodicWatermarks) * @see #assignTimestampsAndWatermarks(AssignerWithPunctuatedWatermarks) */ @Deprecated public SingleOutputStreamOperator<T> assignTimestamps(TimestampExtractor<T> extractor) { // match parallelism to input, otherwise dop=1 sources could lead to some strange // behaviour: the watermark will creep along very slowly because the elements // from the source go to each extraction operator round robin. int inputParallelism = getTransformation().getParallelism(); ExtractTimestampsOperator<T> operator = new ExtractTimestampsOperator<>(clean(extractor)); return transform("ExtractTimestamps", getTransformation().getOutputType(), operator) .setParallelism(inputParallelism); }
final int inputParallelism = getTransformation().getParallelism(); final AssignerWithPunctuatedWatermarks<T> cleanedAssigner = clean(timestampAndWatermarkAssigner);
final int inputParallelism = getTransformation().getParallelism(); final AssignerWithPeriodicWatermarks<T> cleanedAssigner = clean(timestampAndWatermarkAssigner);
Assert.assertEquals(extractOp.getTransformation().getParallelism(), source1.getTransformation().getParallelism());
/** * Gets the parallelism for this operator. * * @return The parallelism set for this operator. */ public int getParallelism() { return transformation.getParallelism(); }
/** * Gets the parallelism for this operator. * * @return The parallelism set for this operator. */ public int getParallelism() { return transformation.getParallelism(); }
/** * Creates a new {@code SplitTransformation} from the given input and {@code OutputSelector}. * * @param input The input {@code StreamTransformation} * @param outputSelector The output selector */ public SplitTransformation(StreamTransformation<T> input, OutputSelector<T> outputSelector) { super("Split", input.getOutputType(), input.getParallelism()); this.input = input; this.outputSelector = outputSelector; }
/** * Creates a new {@code PartitionTransformation} from the given input and * {@link StreamPartitioner}. * * @param input The input {@code StreamTransformation} * @param partitioner The {@code StreamPartitioner} */ public PartitionTransformation(StreamTransformation<T> input, StreamPartitioner<T> partitioner) { super("Partition", input.getOutputType(), input.getParallelism()); this.input = input; this.partitioner = partitioner; }
/** * Creates a new {@code PartitionTransformation} from the given input and * {@link StreamPartitioner}. * * @param input The input {@code StreamTransformation} * @param partitioner The {@code StreamPartitioner} */ public PartitionTransformation(StreamTransformation<T> input, StreamPartitioner<T> partitioner) { super("Partition", input.getOutputType(), input.getParallelism()); this.input = input; this.partitioner = partitioner; }
public SideOutputTransformation(StreamTransformation<?> input, final OutputTag<T> tag) { super("SideOutput", tag.getTypeInfo(), requireNonNull(input).getParallelism()); this.input = input; this.tag = requireNonNull(tag); }
private void calculateBoundedStreamScan(BatchExecBoundedStreamScan boundedStreamScan) { StreamTransformation transformation = boundedStreamScan.getSourceTransformation(tEnv.execEnv()); int parallelism = transformation.getParallelism(); if (parallelism <= 0) { parallelism = tEnv.execEnv().getParallelism(); } finalParallelismNodeMap.put(boundedStreamScan, parallelism); }