Tabnine Logo
StreamExecutionEnvironment.enableCheckpointing
Code IndexAdd Tabnine to your IDE (free)

How to use
enableCheckpointing
method
in
org.apache.flink.streaming.api.environment.StreamExecutionEnvironment

Best Java code snippets using org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.enableCheckpointing (Showing top 20 results out of 315)

origin: apache/flink

/**
 * A thin wrapper layer over {@link StreamExecutionEnvironment#enableCheckpointing(long)}.
 *
 * @param interval Time interval between state checkpoints in milliseconds.
 * @return The same {@code PythonStreamExecutionEnvironment} instance of the caller
 */
public PythonStreamExecutionEnvironment enable_checkpointing(long interval) {
  this.env.enableCheckpointing(interval);
  return this;
}
origin: apache/flink

/**
 * A thin wrapper layer over {@link StreamExecutionEnvironment#enableCheckpointing(long, CheckpointingMode)}.
 *
 * @param interval Time interval between state checkpoints in milliseconds.
 * @param mode The checkpointing mode, selecting between "exactly once" and "at least once" guaranteed.
 * @return The same {@code PythonStreamExecutionEnvironment} instance of the caller
 */
public PythonStreamExecutionEnvironment enable_checkpointing(long interval, CheckpointingMode mode) {
  this.env.enableCheckpointing(interval, mode);
  return this;
}
origin: apache/flink

public static StreamExecutionEnvironment prepareExecutionEnv(ParameterTool parameterTool)
  throws Exception {
  if (parameterTool.getNumberOfParameters() < 5) {
    System.out.println("Missing parameters!\n" +
      "Usage: Kafka --input-topic <topic> --output-topic <topic> " +
      "--bootstrap.servers <kafka brokers> " +
      "--zookeeper.connect <zk quorum> --group.id <some id>");
    throw new Exception("Missing parameters!\n" +
      "Usage: Kafka --input-topic <topic> --output-topic <topic> " +
      "--bootstrap.servers <kafka brokers> " +
      "--zookeeper.connect <zk quorum> --group.id <some id>");
  }
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().disableSysoutLogging();
  env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000));
  env.enableCheckpointing(5000); // create a checkpoint every 5 seconds
  env.getConfig().setGlobalJobParameters(parameterTool); // make parameters available in the web interface
  env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
  return env;
}
origin: apache/flink

public static void main(final String[] args) throws Exception {
  final ParameterTool params = ParameterTool.fromArgs(args);
  final String outputPath = params.getRequired("outputPath");
  final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setParallelism(4);
  env.enableCheckpointing(5000L);
  env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, Time.of(10L, TimeUnit.SECONDS)));
  final StreamingFileSink<Tuple2<Integer, Integer>> sink = StreamingFileSink
    .forRowFormat(new Path(outputPath), (Encoder<Tuple2<Integer, Integer>>) (element, stream) -> {
      PrintStream out = new PrintStream(stream);
      out.println(element.f1);
    })
    .withBucketAssigner(new KeyBucketAssigner())
    .withRollingPolicy(OnCheckpointRollingPolicy.build())
    .build();
  // generate data, shuffle, sink
  env.addSource(new Generator(10, 10, 60))
    .keyBy(0)
    .addSink(sink);
  env.execute("StreamingFileSinkProgram");
}
origin: apache/flink

/**
 * Tests that in a streaming use case where checkpointing is enabled, there is no default strategy set on the
 * client side.
 */
@Test
public void testFallbackStrategyOnClientSideWhenCheckpointingEnabled() throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.enableCheckpointing(500);
  env.fromElements(1).print();
  StreamGraph graph = env.getStreamGraph();
  JobGraph jobGraph = graph.getJobGraph();
  RestartStrategies.RestartStrategyConfiguration restartStrategy =
    jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();
  Assert.assertNotNull(restartStrategy);
  Assert.assertTrue(restartStrategy instanceof RestartStrategies.FallbackRestartStrategyConfiguration);
}
origin: apache/flink

/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 0, restarting is deactivated.
 */
@Test
public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.enableCheckpointing(500);
  env.setNumberOfExecutionRetries(0);
  env.fromElements(1).print();
  StreamGraph graph = env.getStreamGraph();
  JobGraph jobGraph = graph.getJobGraph();
  RestartStrategies.RestartStrategyConfiguration restartStrategy =
    jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();
  Assert.assertNotNull(restartStrategy);
  Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration);
}
origin: apache/flink

/**
 * Runs the following program the test program defined in {@link #testProgram(StreamExecutionEnvironment)}
 * followed by the checks in {@link #postSubmit}.
 */
@Test
public void runCheckpointedProgram() throws Exception {
  try {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(PARALLELISM);
    env.enableCheckpointing(500);
    env.getConfig().disableSysoutLogging();
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 0L));
    testProgram(env);
    TestUtils.tryExecute(env, "Fault Tolerance Test");
    postSubmit();
  }
  catch (Exception e) {
    e.printStackTrace();
    Assert.fail(e.getMessage());
  }
}
origin: apache/flink

public static void main(String[] args) throws Exception {
  ParameterTool params = ParameterTool.fromArgs(args);
  String outputPath = params.getRequired("outputPath");
  StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
  sEnv.setRestartStrategy(RestartStrategies.fixedDelayRestart(
      3,
      Time.of(10, TimeUnit.SECONDS)
    ));
  sEnv.enableCheckpointing(4000);
  final int idlenessMs = 10;
  // define bucketing sink to emit the result
  BucketingSink<Tuple4<Integer, Long, Integer, String>> sink = new BucketingSink<Tuple4<Integer, Long, Integer, String>>(outputPath)
    .setBucketer(new KeyBucketer());
  // generate data, shuffle, perform stateful operation, sink
  sEnv.addSource(new Generator(10, idlenessMs, 60))
    .keyBy(0)
    .map(new SubtractingMapper(-1L * idlenessMs))
    .addSink(sink);
  sEnv.execute();
}
origin: apache/flink

private JobGraph createJobGraph(ExecutionMode mode) {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE);
  env.setRestartStrategy(RestartStrategies.noRestart());
  env.setStateBackend((StateBackend) new MemoryStateBackend());
  switch (mode) {
    case MIGRATE:
      createMigrationJob(env);
      break;
    case RESTORE:
      createRestoredJob(env);
      break;
  }
  return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}
origin: apache/flink

  public static void main(String[] args) throws Exception {
    final ParameterTool pt = ParameterTool.fromArgs(args);
    final String checkpointDir = pt.getRequired("checkpoint.dir");

    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(new FsStateBackend(checkpointDir));
    env.setRestartStrategy(RestartStrategies.noRestart());
    env.enableCheckpointing(1000L);
    env.getConfig().disableGenericTypes();

    env.addSource(new MySource()).uid("my-source")
        .keyBy(anInt -> 0)
        .map(new MyStatefulFunction()).uid("my-map")
        .addSink(new DiscardingSink<>()).uid("my-sink");
    env.execute();
  }
}
origin: apache/flink

public static void main(String[] args) throws Exception {
  ParameterTool params = ParameterTool.fromArgs(args);
  String outputPath = params.getRequired("outputPath");
  int recordsPerSecond = params.getInt("recordsPerSecond", 10);
  int duration = params.getInt("durationInSecond", 60);
  int offset = params.getInt("offsetInSecond", 0);
  StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
  sEnv.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
  sEnv.enableCheckpointing(4000);
  sEnv.getConfig().setAutoWatermarkInterval(1000);
  // execute a simple pass through program.
  PeriodicSourceGenerator generator = new PeriodicSourceGenerator(
    recordsPerSecond, duration, offset);
  DataStream<Tuple> rows = sEnv.addSource(generator);
  DataStream<Tuple> result = rows
    .keyBy(1)
    .timeWindow(Time.seconds(5))
    .sum(0);
  result.writeAsText(outputPath + "/result.txt", FileSystem.WriteMode.OVERWRITE)
    .setParallelism(1);
  sEnv.execute();
}
origin: apache/flink

/**
 * Runs the following program.
 * <pre>
 *     [ (source)->(filter)] -> [ (map) -> (map) ] -> [ (groupBy/reduce)->(sink) ]
 * </pre>
 */
@Override
public void testProgram(StreamExecutionEnvironment env) {
  assertTrue("Broken test setup", NUM_STRINGS % 40 == 0);
  final long failurePosMin = (long) (0.4 * NUM_STRINGS / PARALLELISM);
  final long failurePosMax = (long) (0.7 * NUM_STRINGS / PARALLELISM);
  final long failurePos = (new Random().nextLong() % (failurePosMax - failurePosMin)) + failurePosMin;
  env.enableCheckpointing(200);
  DataStream<String> stream = env.addSource(new StringGeneratingSourceFunction(NUM_STRINGS));
  stream
      // first vertex, chained to the source
      // this filter throttles the flow until at least one checkpoint
      // is complete, to make sure this program does not run without
      .filter(new StringRichFilterFunction())
          // -------------- seconds vertex - one-to-one connected ----------------
      .map(new StringPrefixCountRichMapFunction())
      .startNewChain()
      .map(new StatefulCounterFunction())
          // -------------- third vertex - reducer and the sink ----------------
      .keyBy("prefix")
      .flatMap(new OnceFailingAggregator(failurePos))
      .addSink(new ValidatingSink());
}
origin: apache/flink

private static JobGraph createJobGraphWithOperatorState(
    int parallelism, int maxParallelism, OperatorCheckpointMethod checkpointMethod) {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setParallelism(parallelism);
  env.getConfig().setMaxParallelism(maxParallelism);
  env.enableCheckpointing(Long.MAX_VALUE);
  env.setRestartStrategy(RestartStrategies.noRestart());
  StateSourceBase.workStartedLatch = new CountDownLatch(parallelism);
  SourceFunction<Integer> src;
  switch (checkpointMethod) {
    case CHECKPOINTED_FUNCTION:
      src = new PartitionedStateSource(false);
      break;
    case CHECKPOINTED_FUNCTION_BROADCAST:
      src = new PartitionedStateSource(true);
      break;
    case LIST_CHECKPOINTED:
      src = new PartitionedStateSourceListCheckpointed();
      break;
    case NON_PARTITIONED:
      src = new NonPartitionedStateSource();
      break;
    default:
      throw new IllegalArgumentException();
  }
  DataStream<Integer> input = env.addSource(src);
  input.addSink(new DiscardingSink<Integer>());
  return env.getStreamGraph().getJobGraph();
}
origin: apache/flink

@Test
public void testCheckpointModeTranslation() {
  try {
    // with deactivated fault tolerance, the checkpoint mode should be at-least-once
    StreamExecutionEnvironment deactivated = getSimpleJob();
    for (JobVertex vertex : deactivated.getStreamGraph().getJobGraph().getVertices()) {
      assertEquals(CheckpointingMode.AT_LEAST_ONCE, new StreamConfig(vertex.getConfiguration()).getCheckpointMode());
    }
    // with activated fault tolerance, the checkpoint mode should be by default exactly once
    StreamExecutionEnvironment activated = getSimpleJob();
    activated.enableCheckpointing(1000L);
    for (JobVertex vertex : activated.getStreamGraph().getJobGraph().getVertices()) {
      assertEquals(CheckpointingMode.EXACTLY_ONCE, new StreamConfig(vertex.getConfiguration()).getCheckpointMode());
    }
    // explicitly setting the mode
    StreamExecutionEnvironment explicit = getSimpleJob();
    explicit.enableCheckpointing(1000L, CheckpointingMode.AT_LEAST_ONCE);
    for (JobVertex vertex : explicit.getStreamGraph().getJobGraph().getVertices()) {
      assertEquals(CheckpointingMode.AT_LEAST_ONCE, new StreamConfig(vertex.getConfiguration()).getCheckpointMode());
    }
  }
  catch (Exception e) {
    e.printStackTrace();
    fail(e.getMessage());
  }
}
origin: apache/flink

  /**
   * Checks that in a streaming use case where checkpointing is enabled and the number
   * of execution retries is set to 42 and the delay to 1337, fixed delay restarting is used.
   */
  @Test
  public void testFixedRestartingWhenCheckpointingAndExplicitExecutionRetriesNonZero() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.enableCheckpointing(500);
    env.setNumberOfExecutionRetries(42);
    env.getConfig().setExecutionRetryDelay(1337);

    env.fromElements(1).print();

    StreamGraph graph = env.getStreamGraph();
    JobGraph jobGraph = graph.getJobGraph();

    RestartStrategies.RestartStrategyConfiguration restartStrategy =
      jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

    Assert.assertNotNull(restartStrategy);
    Assert.assertTrue(restartStrategy instanceof RestartStrategies.FixedDelayRestartStrategyConfiguration);
    Assert.assertEquals(42, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getRestartAttempts());
    Assert.assertEquals(1337, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getDelayBetweenAttemptsInterval().toMilliseconds());
  }
}
origin: apache/flink

private static void runPartitioningProgram(int parallelism) throws Exception {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setParallelism(parallelism);
  env.getConfig().enableObjectReuse();
  env.setBufferTimeout(5L);
  env.enableCheckpointing(1000, CheckpointingMode.AT_LEAST_ONCE);
  env
    .addSource(new TimeStampingSource())
    .map(new IdMapper<Tuple2<Long, Long>>())
    .keyBy(0)
    .addSink(new TimestampingSink());
  env.execute("Partitioning Program");
}
origin: apache/flink

private static JobGraph createJobGraphWithKeyedAndNonPartitionedOperatorState(
    int parallelism,
    int maxParallelism,
    int fixedParallelism,
    int numberKeys,
    int numberElements,
    boolean terminateAfterEmission,
    int checkpointingInterval) {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.setParallelism(parallelism);
  env.getConfig().setMaxParallelism(maxParallelism);
  env.enableCheckpointing(checkpointingInterval);
  env.setRestartStrategy(RestartStrategies.noRestart());
  DataStream<Integer> input = env.addSource(new SubtaskIndexNonPartitionedStateSource(
      numberKeys,
      numberElements,
      terminateAfterEmission))
      .setParallelism(fixedParallelism)
      .keyBy(new KeySelector<Integer, Integer>() {
        private static final long serialVersionUID = -7952298871120320940L;
        @Override
        public Integer getKey(Integer value) throws Exception {
          return value;
        }
      });
  SubtaskIndexFlatMapper.workCompletedLatch = new CountDownLatch(numberKeys);
  DataStream<Tuple2<Integer, Integer>> result = input.flatMap(new SubtaskIndexFlatMapper(numberElements));
  result.addSink(new CollectionSink<Tuple2<Integer, Integer>>());
  return env.getStreamGraph().getJobGraph();
}
origin: apache/flink

public static void main(String[] args) throws Exception {
  ParameterTool pt = ParameterTool.fromArgs(args);
  String savepointsPath = pt.getRequired("savepoint-path");
  Configuration config = new Configuration();
  config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointsPath);
  StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(config);
  env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE);
  env.setRestartStrategy(RestartStrategies.noRestart());
  env.setStateBackend(new MemoryStateBackend());
  /**
   * Source -> keyBy -> C(Window -> StatefulMap1 -> StatefulMap2)
   */
  SingleOutputStreamOperator<Tuple2<Integer, Integer>> source = createIntegerTupleSource(env, ExecutionMode.GENERATE);
  SingleOutputStreamOperator<Integer> window = createWindowFunction(ExecutionMode.GENERATE, source);
  SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.GENERATE, window);
  SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.GENERATE, first);
  env.execute("job");
}
origin: apache/flink

@Override
public void testProgram(StreamExecutionEnvironment env) {
  // set the restart strategy.
  env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(NO_OF_RETRIES, 0));
  env.enableCheckpointing(10);
  // create and start the file creating thread.
  fc = new FileCreator();
  fc.start();
  // create the monitoring source along with the necessary readers.
  TextInputFormat format = new TextInputFormat(new org.apache.flink.core.fs.Path(localFsURI));
  format.setFilesFilter(FilePathFilter.createDefaultFilter());
  DataStream<String> inputStream = env.readFile(format, localFsURI,
    FileProcessingMode.PROCESS_CONTINUOUSLY, INTERVAL);
  TestingSinkFunction sink = new TestingSinkFunction();
  inputStream.flatMap(new FlatMapFunction<String, String>() {
    @Override
    public void flatMap(String value, Collector<String> out) throws Exception {
      out.collect(value);
    }
  }).addSink(sink).setParallelism(1);
}
origin: apache/flink

public static void main(String[] args) throws Exception {
  ParameterTool pt = ParameterTool.fromArgs(args);
  String savepointsPath = pt.getRequired("savepoint-path");
  Configuration config = new Configuration();
  config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointsPath);
  StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(config);
  env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE);
  env.setRestartStrategy(RestartStrategies.noRestart());
  env.setStateBackend(new MemoryStateBackend());
  /**
   * Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3)
   */
  DataStream<Integer> source = createSource(env, ExecutionMode.GENERATE);
  SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.GENERATE, source);
  first.startNewChain();
  SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.GENERATE, first);
  second.startNewChain();
  SingleOutputStreamOperator<Integer> stateless = createStatelessMap(second);
  SingleOutputStreamOperator<Integer> third = createThirdStatefulMap(ExecutionMode.GENERATE, stateless);
  env.execute("job");
}
org.apache.flink.streaming.api.environmentStreamExecutionEnvironmentenableCheckpointing

Javadoc

Enables checkpointing for the streaming job. The distributed state of the streaming dataflow will be periodically snapshotted. In case of a failure, the streaming dataflow will be restarted from the latest completed checkpoint. This method selects CheckpointingMode#EXACTLY_ONCE guarantees.

The job draws checkpoints periodically, in the default interval. The state will be stored in the configured state backend.

NOTE: Checkpointing iterative streaming dataflows in not properly supported at the moment. For that reason, iterative jobs will not be started if used with enabled checkpointing. To override this mechanism, use the #enableCheckpointing(long,CheckpointingMode,boolean) method.

Popular methods of StreamExecutionEnvironment

  • execute
  • getExecutionEnvironment
    Creates an execution environment that represents the context in which the program is currently execu
  • addSource
    Ads a data source with a custom type information thus opening a DataStream. Only in very special cas
  • getConfig
    Gets the config object.
  • setStreamTimeCharacteristic
    Sets the time characteristic for all streams create from this environment, e.g., processing time, ev
  • setParallelism
    Sets the parallelism for operations executed through this environment. Setting a parallelism of x he
  • fromElements
    Creates a new data stream that contains the given elements. The elements must all be of the same typ
  • setStateBackend
    Sets the state backend that describes how to store and checkpoint operator state. It defines both wh
  • createLocalEnvironment
    Creates a LocalStreamEnvironment. The local execution environment will run the program in a multi-th
  • fromCollection
    Creates a data stream from the given iterator.Because the iterator will remain unmodified until the
  • getCheckpointConfig
    Gets the checkpoint config, which defines values like checkpoint interval, delay between checkpoints
  • getParallelism
    Gets the parallelism with which operation are executed by default. Operations can individually overr
  • getCheckpointConfig,
  • getParallelism,
  • getStreamGraph,
  • setRestartStrategy,
  • socketTextStream,
  • readTextFile,
  • generateSequence,
  • clean,
  • getStreamTimeCharacteristic

Popular in Java

  • Making http requests using okhttp
  • scheduleAtFixedRate (ScheduledExecutorService)
  • getSharedPreferences (Context)
  • getExternalFilesDir (Context)
  • SQLException (java.sql)
    An exception that indicates a failed JDBC operation. It provides the following information about pro
  • BitSet (java.util)
    The BitSet class implements abit array [http://en.wikipedia.org/wiki/Bit_array]. Each element is eit
  • Executor (java.util.concurrent)
    An object that executes submitted Runnable tasks. This interface provides a way of decoupling task s
  • Annotation (javassist.bytecode.annotation)
    The annotation structure.An instance of this class is returned bygetAnnotations() in AnnotationsAttr
  • SSLHandshakeException (javax.net.ssl)
    The exception that is thrown when a handshake could not be completed successfully.
  • XPath (javax.xml.xpath)
    XPath provides access to the XPath evaluation environment and expressions. Evaluation of XPath Expr
  • Top PhpStorm plugins
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now