/** * A thin wrapper layer over {@link StreamExecutionEnvironment#generateSequence(long, long)}. * * @param from The number to start at (inclusive) * @param to The number to stop at (inclusive) * @return A python data stream, containing all number in the [from, to] interval */ public PythonDataStream generate_sequence(long from, long to) { return new PythonDataStream<>(env.generateSequence(from, to).map(new AdapterMap<>())); }
env.enableCheckpointing(5000); DataStream<String> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1) .map(new MapFunction<Long, String>() { @Override
private static StreamExecutionEnvironment getSimpleJob() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.generateSequence(1, 10000000) .addSink(new SinkFunction<Long>() { @Override public void invoke(Long value) { } }); return env; } }
env.enableCheckpointing(5000); DataStream<Tuple2<String, String>> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1) .flatMap(new FlatMapFunction<Long, Tuple2<String, String>>() { @Override
env.enableCheckpointing(5000); DataStream<Tuple2<String, String>> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1) .flatMap(new FlatMapFunction<Long, Tuple2<String, String>>() { @Override
env.enableCheckpointing(5000); DataStream<Tuple2<String, String>> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1) .flatMap(new FlatMapFunction<Long, Tuple2<String, String>>() { @Override
@Test public void testCollect() throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); final long n = 10; DataStream<Long> stream = env.generateSequence(1, n); long i = 1; for (Iterator<Long> it = DataStreamUtils.collect(stream); it.hasNext(); ) { long x = it.next(); assertEquals("received wrong element", i, x); i++; } assertEquals("received wrong number of elements", n + 1, i); } }
env.enableCheckpointing(5000); DataStream<Tuple2<String, String>> source = env.generateSequence(0, parameterTool.getInt("numRecords") - 1) .flatMap(new FlatMapFunction<Long, Tuple2<String, String>>() { @Override
DataStream<Long> src1 = env.generateSequence(1, 10); DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("src-1"); DataStream<Long> src3 = env.generateSequence(1, 10).slotSharingGroup("group-1"); DataStream<Long> src4 = env.generateSequence(1, 10).slotSharingGroup("group-1");
@Test public void testUnion() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); FilterFunction<Long> dummyFilter = new FilterFunction<Long>() { @Override public boolean filter(Long value) { return false; } }; DataStream<Long> src1 = env.generateSequence(1, 10); DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("src-1"); // this should not inherit group "src-1" src1.union(src2).filter(dummyFilter); DataStream<Long> src3 = env.generateSequence(1, 10).slotSharingGroup("group-1"); DataStream<Long> src4 = env.generateSequence(1, 10).slotSharingGroup("group-1"); // this should inherit "group-1" now src3.union(src4).filter(dummyFilter); JobGraph jobGraph = env.getStreamGraph().getJobGraph(); List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources(); // first pipeline assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup()); assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup()); assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup()); // second pipeline assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(3).getSlotSharingGroup()); assertEquals(vertices.get(2).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup()); assertEquals(vertices.get(3).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup()); }
@Test public void testInheritOverride() { // verify that we can explicitly disable inheritance of the input slot sharing groups StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); FilterFunction<Long> dummyFilter = new FilterFunction<Long>() { @Override public boolean filter(Long value) { return false; } }; DataStream<Long> src1 = env.generateSequence(1, 10).slotSharingGroup("group-1"); DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("group-1"); // this should not inherit group but be in "default" src1.union(src2).filter(dummyFilter).slotSharingGroup("default"); JobGraph jobGraph = env.getStreamGraph().getJobGraph(); List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources(); assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup()); assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup()); assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup()); }
private static void addSmallBoundedJob(StreamExecutionEnvironment env, int parallelism) { DataStream<Long> stream = env.generateSequence(1, 100).setParallelism(parallelism); stream .filter(ignored -> false).setParallelism(parallelism) .startNewChain() .print().setParallelism(parallelism); } }
public void testKeyedStreamProcessTranslation() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStreamSource<Long> src = env.generateSequence(0, 0);
@Test public void outputSelectorTest() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(3); TestListResultSink<Long> evenSink = new TestListResultSink<Long>(); TestListResultSink<Long> oddAndTenSink = new TestListResultSink<Long>(); TestListResultSink<Long> evenAndOddSink = new TestListResultSink<Long>(); TestListResultSink<Long> allSink = new TestListResultSink<Long>(); SplitStream<Long> source = env.generateSequence(1, 11).split(new MyOutputSelector()); source.select(EVEN).addSink(evenSink); source.select(ODD, TEN).addSink(oddAndTenSink); source.select(EVEN, ODD).addSink(evenAndOddSink); source.addSink(allSink); env.execute(); assertEquals(Arrays.asList(2L, 4L, 6L, 8L, 10L), evenSink.getSortedResult()); assertEquals(Arrays.asList(1L, 3L, 5L, 7L, 9L, 10L, 11L), oddAndTenSink.getSortedResult()); assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L), evenAndOddSink.getSortedResult()); assertEquals(Arrays.asList(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L, 9L, 10L, 11L), allSink.getSortedResult()); } }
/** * Verify that a {@link DataStream#process(ProcessFunction)} call is correctly translated to an operator. */ @Test public void testProcessTranslation() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStreamSource<Long> src = env.generateSequence(0, 0); ProcessFunction<Long, Integer> processFunction = new ProcessFunction<Long, Integer>() { private static final long serialVersionUID = 1L; @Override public void processElement( Long value, Context ctx, Collector<Integer> out) throws Exception { // Do nothing } @Override public void onTimer( long timestamp, OnTimerContext ctx, Collector<Integer> out) throws Exception { // Do nothing } }; DataStream<Integer> processed = src .process(processFunction); processed.addSink(new DiscardingSink<Integer>()); assertEquals(processFunction, getFunctionForDataStream(processed)); assertTrue(getOperatorForDataStream(processed) instanceof ProcessOperator); }
@Test public void testSources() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); SourceFunction<Integer> srcFun = new SourceFunction<Integer>() { private static final long serialVersionUID = 1L; @Override public void run(SourceContext<Integer> ctx) throws Exception { } @Override public void cancel() { } }; DataStreamSource<Integer> src1 = env.addSource(srcFun); src1.addSink(new DiscardingSink<Integer>()); assertEquals(srcFun, getFunctionFromDataSource(src1)); List<Long> list = Arrays.asList(0L, 1L, 2L); DataStreamSource<Long> src2 = env.generateSequence(0, 2); assertTrue(getFunctionFromDataSource(src2) instanceof StatefulSequenceSource); DataStreamSource<Long> src3 = env.fromElements(0L, 1L, 2L); assertTrue(getFunctionFromDataSource(src3) instanceof FromElementsFunction); DataStreamSource<Long> src4 = env.fromCollection(list); assertTrue(getFunctionFromDataSource(src4) instanceof FromElementsFunction); }
/** * Verify that a {@link KeyedStream#process(KeyedProcessFunction)} call is correctly translated to an operator. */ @Test public void testKeyedStreamKeyedProcessTranslation() { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStreamSource<Long> src = env.generateSequence(0, 0); KeyedProcessFunction<Long, Long, Integer> keyedProcessFunction = new KeyedProcessFunction<Long, Long, Integer>() { private static final long serialVersionUID = 1L; @Override public void processElement(Long value, Context ctx, Collector<Integer> out) throws Exception { // Do nothing } @Override public void onTimer(long timestamp, OnTimerContext ctx, Collector<Integer> out) throws Exception { // Do nothing } }; DataStream<Integer> processed = src .keyBy(new IdentityKeySelector<Long>()) .process(keyedProcessFunction); processed.addSink(new DiscardingSink<Integer>()); assertEquals(keyedProcessFunction, getFunctionForDataStream(processed)); assertTrue(getOperatorForDataStream(processed) instanceof KeyedProcessOperator); }
final DataStream<Long> srcOne = env.generateSequence(0L, 5L) .assignTimestampsAndWatermarks(new CustomWmEmitter<Long>() {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStreamSink<Long> sink = env.generateSequence(1, 100).print(); assertTrue(env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getStatePartitioner1() == null); assertTrue(env.getStreamGraph().getStreamNode(sink.getTransformation().getId()).getInEdges().get(0).getPartitioner() instanceof ForwardPartitioner); DataStreamSink<Long> sink2 = env.generateSequence(1, 100).keyBy(key1).print(); DataStreamSink<Long> sink3 = env.generateSequence(1, 100).keyBy(key2).print();
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); DataStream<Long> src1 = env.generateSequence(0, 0); assertEquals(TypeExtractor.getForClass(Long.class), src1.getType());