/** * A thin wrapper layer over {@link StreamExecutionEnvironment#enableCheckpointing(long)}. * * @param interval Time interval between state checkpoints in milliseconds. * @return The same {@code PythonStreamExecutionEnvironment} instance of the caller */ public PythonStreamExecutionEnvironment enable_checkpointing(long interval) { this.env.enableCheckpointing(interval); return this; }
public int getParallelism() { if (parallelism == ExecutionConfig.PARALLELISM_DEFAULT) { return env.getParallelism(); } else { return parallelism; } }
/** * Returns the checkpointing interval or -1 if checkpointing is disabled. * * <p>Shorthand for {@code getCheckpointConfig().getCheckpointInterval()}. * * @return The checkpointing interval or -1 */ public long getCheckpointInterval() { return checkpointCfg.getCheckpointInterval(); }
/** * Triggers the program execution. The environment will execute all parts of * the program that have resulted in a "sink" operation. Sink operations are * for example printing results or forwarding them to a message queue. * * <p>The program execution will be logged and displayed with a generated * default name. * * @return The result of the job execution, containing elapsed time and accumulators. * @throws Exception which occurs during job execution. */ public JobExecutionResult execute() throws Exception { return execute(DEFAULT_JOB_NAME); }
@Test public void testCheckpointConfigDefault() throws Exception { StreamExecutionEnvironment streamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment(); Assert.assertTrue(streamExecutionEnvironment.getCheckpointConfig().isFailOnCheckpointingErrors()); }
@Override public <OUT> DataStreamSource<OUT> addSource(SourceFunction<OUT> sourceFunction) { this.sourceFunction = sourceFunction; return super.addSource(sourceFunction); }
public StreamGraph(StreamExecutionEnvironment environment) { this.environment = environment; this.executionConfig = environment.getConfig(); this.checkpointConfig = environment.getCheckpointConfig(); // create an empty new stream graph. clear(); }
public ExecutionConfig getExecutionConfig() { return environment.getConfig(); }
/** * A thin wrapper layer over {@link StreamExecutionEnvironment#setParallelism(int)}. * * @param parallelism The parallelism * @return The same {@code PythonStreamExecutionEnvironment} instance of the caller */ public PythonStreamExecutionEnvironment set_parallelism(int parallelism) { this.env.setParallelism(parallelism); return this; }
/** * Creates a {@link LocalStreamEnvironment}. The local execution environment * will run the program in a multi-threaded fashion in the same JVM as the * environment was created in. The default parallelism of the local * environment is the number of hardware contexts (CPU cores / threads), * unless it was specified differently by {@link #setParallelism(int)}. * * @return A local execution environment. */ public static LocalStreamEnvironment createLocalEnvironment() { return createLocalEnvironment(defaultLocalParallelism); }
public <F> F clean(F f) { return environment.clean(f); }
/** * Test test verifies that the execution environment can be used to execute a * single job with multiple slots. */ @Test public void testRunIsolatedJob() throws Exception { LocalStreamEnvironment env = new LocalStreamEnvironment(); assertEquals(1, env.getParallelism()); addSmallBoundedJob(env, 3); env.execute(); }
/** * Test test verifies that the execution environment can be used to execute multiple * bounded streaming jobs after one another. */ @Test public void testMultipleJobsAfterAnother() throws Exception { LocalStreamEnvironment env = new LocalStreamEnvironment(); addSmallBoundedJob(env, 3); env.execute(); addSmallBoundedJob(env, 5); env.execute(); }
/** * Returns the checkpointing mode (exactly-once vs. at-least-once). * * <p>Shorthand for {@code getCheckpointConfig().getCheckpointingMode()}. * * @return The checkpoint mode */ public CheckpointingMode getCheckpointingMode() { return checkpointCfg.getCheckpointingMode(); }
private static void registerJythonSerializers(StreamExecutionEnvironment env) { env.registerTypeWithKryoSerializer(PyBoolean.class, PyBooleanSerializer.class); env.registerTypeWithKryoSerializer(PyFloat.class, PyFloatSerializer.class); env.registerTypeWithKryoSerializer(PyInteger.class, PyIntegerSerializer.class); env.registerTypeWithKryoSerializer(PyLong.class, PyLongSerializer.class); env.registerTypeWithKryoSerializer(PyString.class, PyStringSerializer.class); env.registerTypeWithKryoSerializer(PyUnicode.class, PyObjectSerializer.class); env.registerTypeWithKryoSerializer(PyTuple.class, PyObjectSerializer.class); env.registerTypeWithKryoSerializer(PyObjectDerived.class, PyObjectSerializer.class); env.registerTypeWithKryoSerializer(PyInstance.class, PyObjectSerializer.class); }
@Override public String toString() { return "Remote Environment (" + this.host + ":" + this.port + " - parallelism = " + (getParallelism() == -1 ? "default" : getParallelism()) + ")"; }
@Override public JobExecutionResult execute() throws Exception { return execute(""); }
public static void resetContextEnvironments() { StreamExecutionEnvironment.resetContextEnvironment(); } }
public Long getBufferTimeout() { return bufferTimeout != null ? bufferTimeout : env.getBufferTimeout(); }
/** * A thin wrapper layer over {@link StreamExecutionEnvironment#enableCheckpointing(long, CheckpointingMode)}. * * @param interval Time interval between state checkpoints in milliseconds. * @param mode The checkpointing mode, selecting between "exactly once" and "at least once" guaranteed. * @return The same {@code PythonStreamExecutionEnvironment} instance of the caller */ public PythonStreamExecutionEnvironment enable_checkpointing(long interval, CheckpointingMode mode) { this.env.enableCheckpointing(interval, mode); return this; }