org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.setStateBackend java code examples

Common ways to obtain StreamExecutionEnvironment

private void myMethod () {
 StreamExecutionEnvironment s =StreamExecutionEnvironment.getExecutionEnvironment()
StreamExecutionEnvironment.createLocalEnvironment()
Smart code suggestions by Tabnine
}

  /**
   * Sets the state backend to a new {@link StubStateBackend} which has a {@link MonotonicTTLTimeProvider}.
   *
   * @param env The {@link StreamExecutionEnvironment} of the job.
   * @return The {@link MonotonicTTLTimeProvider}.
   */
  private static MonotonicTTLTimeProvider setBackendWithCustomTTLTimeProvider(StreamExecutionEnvironment env) {
    final MonotonicTTLTimeProvider ttlTimeProvider = new MonotonicTTLTimeProvider();

    final StateBackend configuredBackend = env.getStateBackend();
    final StateBackend stubBackend = new StubStateBackend(configuredBackend, ttlTimeProvider);
    env.setStateBackend(stubBackend);

    return ttlTimeProvider;
  }
}

  env.setStateBackend(new FsStateBackend(checkpointDir, asyncCheckpoints));
} else if ("rocks".equals(stateBackend)) {
  boolean incrementalCheckpoints = pt.getBoolean("incrementalCheckpoints", false);
  env.setStateBackend(new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
} else {
  throw new IllegalArgumentException("Unknown backend: " + stateBackend);

env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 0L));
env.setStateBackend(stateBackend);

env.setStateBackend(stateBackend);
env.enableCheckpointing(1000L);
env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);

private JobGraph createJobGraph(ExecutionMode mode) {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE);
  env.setRestartStrategy(RestartStrategies.noRestart());
  env.setStateBackend((StateBackend) new MemoryStateBackend());
  switch (mode) {
    case MIGRATE:
      createMigrationJob(env);
      break;
    case RESTORE:
      createRestoredJob(env);
      break;
  }
  return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}

  public static void main(String[] args) throws Exception {
    final ParameterTool pt = ParameterTool.fromArgs(args);
    final String checkpointDir = pt.getRequired("checkpoint.dir");

    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(new FsStateBackend(checkpointDir));
    env.setRestartStrategy(RestartStrategies.noRestart());
    env.enableCheckpointing(1000L);
    env.getConfig().disableGenericTypes();

    env.addSource(new MySource()).uid("my-source")
        .keyBy(anInt -> 0)
        .map(new MyStatefulFunction()).uid("my-map")
        .addSink(new DiscardingSink<>()).uid("my-sink");
    env.execute();
  }
}

env.enableCheckpointing(200);
env.setStateBackend(new FsStateBackend(tempCheckpointDir.getAbsoluteFile().toURI()));

env.setStateBackend(stateBackend);
env.setParallelism(maxParallelism);

  env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
  break;
case StateBackendLoader.MEMORY_STATE_BACKEND_NAME:
  env.setStateBackend(new MemoryStateBackend());
  break;
default:

see.setStateBackend(new FailingStateBackend());

public static void main(String[] args) throws Exception {
  ParameterTool pt = ParameterTool.fromArgs(args);
  String savepointsPath = pt.getRequired("savepoint-path");
  Configuration config = new Configuration();
  config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointsPath);
  StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(config);
  env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE);
  env.setRestartStrategy(RestartStrategies.noRestart());
  env.setStateBackend(new MemoryStateBackend());
  /**
   * Source -> keyBy -> C(Window -> StatefulMap1 -> StatefulMap2)
   */
  SingleOutputStreamOperator<Tuple2<Integer, Integer>> source = createIntegerTupleSource(env, ExecutionMode.GENERATE);
  SingleOutputStreamOperator<Integer> window = createWindowFunction(ExecutionMode.GENERATE, source);
  SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.GENERATE, window);
  SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.GENERATE, first);
  env.execute("job");
}

env.setStateBackend(stateBackend);
env.setParallelism(maxParallelism);

env.setStateBackend(stateBackend);
env.setParallelism(maxParallelism);

env.setStateBackend(stateBackend);
env.setParallelism(maxParallelism);

public static void main(String[] args) throws Exception {
  ParameterTool pt = ParameterTool.fromArgs(args);
  String savepointsPath = pt.getRequired("savepoint-path");
  Configuration config = new Configuration();
  config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointsPath);
  StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(config);
  env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE);
  env.setRestartStrategy(RestartStrategies.noRestart());
  env.setStateBackend(new MemoryStateBackend());
  /**
   * Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3)
   */
  DataStream<Integer> source = createSource(env, ExecutionMode.GENERATE);
  SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.GENERATE, source);
  first.startNewChain();
  SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.GENERATE, first);
  second.startNewChain();
  SingleOutputStreamOperator<Integer> stateless = createStatelessMap(second);
  SingleOutputStreamOperator<Integer> third = createThirdStatefulMap(ExecutionMode.GENERATE, stateless);
  env.execute("job");
}

env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);

env.setStateBackend(stateBackend);
env.setParallelism(maxParallelism);

env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
env.getConfig().disableSysoutLogging();
env.setStateBackend(this.stateBackend);
env.getConfig().setUseSnapshotCompression(true);

  env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
  break;
case StateBackendLoader.MEMORY_STATE_BACKEND_NAME:
  env.setStateBackend(new MemoryStateBackend());
  break;
default:

  env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
  break;
case StateBackendLoader.MEMORY_STATE_BACKEND_NAME:
  env.setStateBackend(new MemoryStateBackend());
  break;
default:

Popular methods of StreamExecutionEnvironment

execute
getExecutionEnvironment
Creates an execution environment that represents the context in which the program is currently execu
addSource
Ads a data source with a custom type information thus opening a DataStream. Only in very special cas
getConfig
Gets the config object.
enableCheckpointing
Enables checkpointing for the streaming job. The distributed state of the streaming dataflow will be
setStreamTimeCharacteristic
Sets the time characteristic for all streams create from this environment, e.g., processing time, ev
setParallelism
Sets the parallelism for operations executed through this environment. Setting a parallelism of x he
fromElements
Creates a new data stream that contains the given elements. The elements must all be of the same typ
createLocalEnvironment
Creates a LocalStreamEnvironment. The local execution environment will run the program in a multi-th
fromCollection
Creates a data stream from the given iterator.Because the iterator will remain unmodified until the
getCheckpointConfig
Gets the checkpoint config, which defines values like checkpoint interval, delay between checkpoints
getParallelism
Gets the parallelism with which operation are executed by default. Operations can individually overr

Popular in Java

Making http requests using okhttp
onCreateOptionsMenu (Activity)
getSharedPreferences (Context)
findViewById (Activity)
Menu (java.awt)
Timestamp (java.sql)
A Java representation of the SQL TIMESTAMP type. It provides the capability of representing the SQL
LinkedHashMap (java.util)
LinkedHashMap is an implementation of Map that guarantees iteration order. All optional operations a
AtomicInteger (java.util.concurrent.atomic)
An int value that may be updated atomically. See the java.util.concurrent.atomic package specificati
JLabel (javax.swing)
Logger (org.slf4j)
The org.slf4j.Logger interface is the main user entry point of SLF4J API. It is expected that loggin
PhpStorm for WordPress

How to use setStateBackendmethodin org.apache.flink.streaming.api.environment.StreamExecutionEnvironment

Best Java code snippets using org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.setStateBackend (Showing top 20 results out of 315)

How to use
setStateBackend
method
in
org.apache.flink.streaming.api.environment.StreamExecutionEnvironment