/** * A thin wrapper layer over {@link DataStream#union(DataStream[])}. * * @param streams The Python DataStreams to union output with. * @return The {@link PythonDataStream}. */ @SafeVarargs @SuppressWarnings("unchecked") public final PythonDataStream union(PythonDataStream... streams) { ArrayList<DataStream<PyObject>> dsList = new ArrayList<>(); for (PythonDataStream ps : streams) { dsList.add(ps.stream); } DataStream<PyObject>[] dsArray = new DataStream[dsList.size()]; return new PythonDataStream(stream.union(dsList.toArray(dsArray))); }
@Test public void testUnion() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); FilterFunction<Long> dummyFilter = new FilterFunction<Long>() { @Override public boolean filter(Long value) { return false; } }; DataStream<Long> src1 = env.generateSequence(1, 10); DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("src-1"); // this should not inherit group "src-1" src1.union(src2).filter(dummyFilter); DataStream<Long> src3 = env.generateSequence(1, 10).slotSharingGroup("group-1"); DataStream<Long> src4 = env.generateSequence(1, 10).slotSharingGroup("group-1"); // this should inherit "group-1" now src3.union(src4).filter(dummyFilter); JobGraph jobGraph = env.getStreamGraph().getJobGraph(); List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources(); // first pipeline assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup()); assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup()); assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup()); // second pipeline assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(3).getSlotSharingGroup()); assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup()); assertEquals(vertices.get(3).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup()); }
@Test public void testInheritOverride() { // verify that we can explicitly disable inheritance of the input slot sharing groups StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); FilterFunction<Long> dummyFilter = new FilterFunction<Long>() { @Override public boolean filter(Long value) { return false; } }; DataStream<Long> src1 = env.generateSequence(1, 10).slotSharingGroup("group-1"); DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("group-1"); // this should not inherit group but be in "default" src1.union(src2).filter(dummyFilter).slotSharingGroup("default"); JobGraph jobGraph = env.getStreamGraph().getJobGraph(); List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources(); assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup()); assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup()); assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup()); }
DataStream<Integer> d2 = env.fromElements(1, 3, 5, 7, 9); d1 = d1.union(d2);
new IntGeneratingSourceFunction(NUM_STRINGS / 2, NUM_STRINGS / 4)); stream1.union(stream2) .keyBy(new IdentityKeySelector<Integer>()) .map(new OnceFailingPartitionedSum(failurePos))
DataStream<Integer> source2 = env.addSource(new MyTimestampSourceInfinite(initialTime, numWatermarks / 2)); source1.union(source2) .map(new IdentityMap()) .connect(source2).map(new IdentityCoMap())
/** * Tests that there are no collisions with two identical sources. * * <pre> * [ (src0) ] --\ * +--> [ (sink) ] * [ (src1) ] --/ * </pre> */ @Test public void testNodeHashIdenticalSources() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(); env.setParallelism(4); env.disableOperatorChaining(); DataStream<String> src0 = env.addSource(new NoOpSourceFunction()); DataStream<String> src1 = env.addSource(new NoOpSourceFunction()); src0.union(src1).addSink(new NoOpSinkFunction()); JobGraph jobGraph = env.getStreamGraph().getJobGraph(); List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources(); assertTrue(vertices.get(0).isInputVertex()); assertTrue(vertices.get(1).isInputVertex()); assertNotNull(vertices.get(0).getID()); assertNotNull(vertices.get(1).getID()); assertNotEquals(vertices.get(0).getID(), vertices.get(1).getID()); }
DataStream<Integer> source2 = env.addSource(new MyTimestampSource(initialTime, numWatermarks / 2)); source1.union(source2) .map(new IdentityMap()) .connect(source2).map(new IdentityCoMap())
@Test public void testUnionBetweenConsecutiveSplitRejection() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStreamSource<Integer> src = env.fromElements(0, 0); OutputSelector<Integer> outputSelector = new DummyOutputSelector<>(); src.split(outputSelector).select("dummy").union(src.map(x -> x)).split(outputSelector).addSink(new DiscardingSink<>()); expectedException.expect(IllegalStateException.class); expectedException.expectMessage("Consecutive multiple splits are not supported. Splits are deprecated. Please use side-outputs."); env.getStreamGraph(); }
coIt.closeWith(head.broadcast().union(otherSource));
.returns(unionType); DataStream<TaggedUnion<T1, T2>> unionStream = taggedInput1.union(taggedInput2);
DataStream<Integer> unionFilter = map.union(flatMap) .filter(filterFunction);
DataStream<Long> selfUnion = input1.union(input1).map(new MapFunction<Long, Long>() { @Override public Long map(Long value) throws Exception { DataStream<Long> selfUnionDifferentPartition = input6.broadcast().union(input6).map(new MapFunction<Long, Long>() { @Override public Long map(Long value) throws Exception { DataStream<Long> unionDifferingParallelism = input2.union(input3).map(new MapFunction<Long, Long>() { @Override public Long map(Long value) throws Exception { DataStream<Long> unionDifferingPartitioning = input4.broadcast().union(input5).map(new MapFunction<Long, Long>() { @Override public Long map(Long value) throws Exception {
@Override public void translateNode(Flatten.FlattenPCollectionList<T> transform, FlinkStreamingTranslationContext context) { List<PCollection<T>> allInputs = context.getInput(transform).getAll(); DataStream<T> result = null; for (PCollection<T> collection : allInputs) { DataStream<T> current = context.getInputDataStream(collection); result = (result == null) ? current : result.union(current); } context.setOutputDataStream(context.getOutput(transform), result); } }
@Override protected DataStream<Tuple2<StreamRoute, Object>> toDataStream() { final String localFirstStreamId = firstStreamId; final List<String> localUnionStreamIds = this.unionStreamIds; DataStream<Tuple2<StreamRoute, Object>> dataStream = convertDataStream(getCepEnvironment().<T>getDataStream(localFirstStreamId), this.firstStreamId); for (String unionStreamId : localUnionStreamIds) { dataStream = dataStream.union(convertDataStream(getCepEnvironment().<T>getDataStream(unionStreamId), unionStreamId)); } return dataStream; } }