/** * Sets the state backend to a new {@link StubStateBackend} which has a {@link MonotonicTTLTimeProvider}. * * @param env The {@link StreamExecutionEnvironment} of the job. * @return The {@link MonotonicTTLTimeProvider}. */ private static MonotonicTTLTimeProvider setBackendWithCustomTTLTimeProvider(StreamExecutionEnvironment env) { final MonotonicTTLTimeProvider ttlTimeProvider = new MonotonicTTLTimeProvider(); final StateBackend configuredBackend = env.getStateBackend(); final StateBackend stubBackend = new StubStateBackend(configuredBackend, ttlTimeProvider); env.setStateBackend(stubBackend); return ttlTimeProvider; } }
env.setStateBackend(new FsStateBackend(checkpointDir, asyncCheckpoints)); } else if ("rocks".equals(stateBackend)) { boolean incrementalCheckpoints = pt.getBoolean("incrementalCheckpoints", false); env.setStateBackend(new RocksDBStateBackend(checkpointDir, incrementalCheckpoints)); } else { throw new IllegalArgumentException("Unknown backend: " + stateBackend);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 0L)); env.setStateBackend(stateBackend);
env.setStateBackend(stateBackend); env.enableCheckpointing(1000L); env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
private JobGraph createJobGraph(ExecutionMode mode) { StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE); env.setRestartStrategy(RestartStrategies.noRestart()); env.setStateBackend((StateBackend) new MemoryStateBackend()); switch (mode) { case MIGRATE: createMigrationJob(env); break; case RESTORE: createRestoredJob(env); break; } return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph()); }
public static void main(String[] args) throws Exception { final ParameterTool pt = ParameterTool.fromArgs(args); final String checkpointDir = pt.getRequired("checkpoint.dir"); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStateBackend(new FsStateBackend(checkpointDir)); env.setRestartStrategy(RestartStrategies.noRestart()); env.enableCheckpointing(1000L); env.getConfig().disableGenericTypes(); env.addSource(new MySource()).uid("my-source") .keyBy(anInt -> 0) .map(new MyStatefulFunction()).uid("my-map") .addSink(new DiscardingSink<>()).uid("my-sink"); env.execute(); } }
env.enableCheckpointing(200); env.setStateBackend(new FsStateBackend(tempCheckpointDir.getAbsoluteFile().toURI()));
env.setStateBackend(stateBackend); env.setParallelism(maxParallelism);
env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend())); break; case StateBackendLoader.MEMORY_STATE_BACKEND_NAME: env.setStateBackend(new MemoryStateBackend()); break; default:
see.setStateBackend(new FailingStateBackend());
public static void main(String[] args) throws Exception { ParameterTool pt = ParameterTool.fromArgs(args); String savepointsPath = pt.getRequired("savepoint-path"); Configuration config = new Configuration(); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointsPath); StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(config); env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE); env.setRestartStrategy(RestartStrategies.noRestart()); env.setStateBackend(new MemoryStateBackend()); /** * Source -> keyBy -> C(Window -> StatefulMap1 -> StatefulMap2) */ SingleOutputStreamOperator<Tuple2<Integer, Integer>> source = createIntegerTupleSource(env, ExecutionMode.GENERATE); SingleOutputStreamOperator<Integer> window = createWindowFunction(ExecutionMode.GENERATE, source); SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.GENERATE, window); SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.GENERATE, first); env.execute("job"); }
env.setStateBackend(stateBackend); env.setParallelism(maxParallelism);
env.setStateBackend(stateBackend); env.setParallelism(maxParallelism);
env.setStateBackend(stateBackend); env.setParallelism(maxParallelism);
public static void main(String[] args) throws Exception { ParameterTool pt = ParameterTool.fromArgs(args); String savepointsPath = pt.getRequired("savepoint-path"); Configuration config = new Configuration(); config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointsPath); StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(config); env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE); env.setRestartStrategy(RestartStrategies.noRestart()); env.setStateBackend(new MemoryStateBackend()); /** * Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3) */ DataStream<Integer> source = createSource(env, ExecutionMode.GENERATE); SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.GENERATE, source); first.startNewChain(); SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.GENERATE, first); second.startNewChain(); SingleOutputStreamOperator<Integer> stateless = createStatelessMap(second); SingleOutputStreamOperator<Integer> third = createThirdStatefulMap(ExecutionMode.GENERATE, stateless); env.execute("job"); }
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0)); env.getConfig().disableSysoutLogging(); env.setStateBackend(this.stateBackend); env.getConfig().setUseSnapshotCompression(true);
env.setStateBackend(stateBackend); env.setParallelism(maxParallelism);
env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0)); env.getConfig().disableSysoutLogging(); env.setStateBackend(this.stateBackend); env.getConfig().setUseSnapshotCompression(true);
env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend())); break; case StateBackendLoader.MEMORY_STATE_BACKEND_NAME: env.setStateBackend(new MemoryStateBackend()); break; default:
env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend())); break; case StateBackendLoader.MEMORY_STATE_BACKEND_NAME: env.setStateBackend(new MemoryStateBackend()); break; default: