private static void runPartitioningProgram(int parallelism) throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(parallelism); env.getConfig().enableObjectReuse(); env.setBufferTimeout(5L); env.enableCheckpointing(1000, CheckpointingMode.AT_LEAST_ONCE); env .addSource(new TimeStampingSource()) .map(new IdMapper<Tuple2<Long, Long>>()) .keyBy(0) .addSink(new TimestampingSink()); env.execute("Partitioning Program"); }
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setBufferTimeout(77); // set timeout to some recognizable number
@Override public void processStream(final FlinkBenchConfig config) throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setBufferTimeout(config.bufferTimeout); createDataStream(config); DataStream<Tuple2<String, String>> dataStream = env.addSource(getDataStream()); dataStream.map(new MapFunction<Tuple2<String, String>, Tuple2<String, String>>() { @Override public Tuple2<String, String> map(Tuple2<String, String> value) throws Exception { KafkaReporter kafkaReporter = new KafkaReporter(config.reportTopic, config.brokerList); kafkaReporter.report(Long.parseLong(value.f0), System.currentTimeMillis()); return value; } }); env.execute("Identity Job"); } }
@Override public void processStream(final FlinkBenchConfig config) throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setBufferTimeout(config.bufferTimeout); createDataStream(config); DataStream<Tuple2<String, String>> dataStream = env.addSource(getDataStream()); dataStream.rebalance().map( new MapFunction<Tuple2<String, String>, Tuple2<String, String>>() { @Override public Tuple2<String, String> map(Tuple2<String, String> value) throws Exception { KafkaReporter kafkaReporter = new KafkaReporter(config.reportTopic, config.brokerList); kafkaReporter.report(Long.parseLong(value.f0), System.currentTimeMillis()); return value; } }); env.execute("Repartition Job"); } }
env.setBufferTimeout(config.bufferTimeout); env.enableCheckpointing(config.checkpointDuration);
@Override public void processStream(final FlinkBenchConfig config) throws Exception { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setBufferTimeout(config.bufferTimeout); env.enableCheckpointing(config.checkpointDuration); createDataStream(config);
env.setBufferTimeout(FlinkUtil.getBufferTimeoutMillis(confProperties));
private StreamExecutionEnvironment createJobGraph(int parallelism, Function<DataStream<String>, DataStream<String>> pipelinerBuilder) throws Exception { final Path checkpointDir = getCheckpointDir(); final Path savepointRootDir = getSavepointDir(); checkpointDir.getFileSystem().mkdirs(checkpointDir); savepointRootDir.getFileSystem().mkdirs(savepointRootDir); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableSysoutLogging(); env.getCheckpointConfig().enableExternalizedCheckpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION); env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime); env.setBufferTimeout(0); env.setParallelism(parallelism); env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE); env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir.toString(), true)); DataStream<String> sourceData = env .addSource(new TestPipelineSource()) .uid("TestSource") .name("TestSource") .setParallelism(1); pipelinerBuilder.apply(sourceData) .addSink(new CollectingSink()).name("Output").uid("Output") .setParallelism(1); return env; }