public int getParallelism() { if (parallelism == ExecutionConfig.PARALLELISM_DEFAULT) { return env.getParallelism(); } else { return parallelism; } }
public DataStreamSource(StreamExecutionEnvironment environment, TypeInformation<T> outTypeInfo, StreamSource<T, ?> operator, boolean isParallel, String sourceName) { super(environment, new SourceTransformation<>(sourceName, operator, outTypeInfo, environment.getParallelism())); this.isParallel = isParallel; if (!isParallel) { setParallelism(1); } }
@SuppressWarnings("unchecked") protected DataStreamSink(DataStream<T> inputStream, StreamSink<T> operator) { this.transformation = new SinkTransformation<T>(inputStream.getTransformation(), "Unnamed", operator, inputStream.getExecutionEnvironment().getParallelism()); }
/** * Method for passing user defined operators along with the type * information that will transform the DataStream. * * @param operatorName * name of the operator, for logging purposes * @param outTypeInfo * the output type of the operator * @param operator * the object containing the transformation logic * @param <R> * type of the return stream * @return the data stream constructed */ @PublicEvolving public <R> SingleOutputStreamOperator<R> transform(String operatorName, TypeInformation<R> outTypeInfo, OneInputStreamOperator<T, R> operator) { // read the output type of the input Transform to coax out errors about MissingTypeInfo transformation.getOutputType(); OneInputTransformation<T, R> resultTransform = new OneInputTransformation<>( this.transformation, operatorName, operator, outTypeInfo, environment.getParallelism()); @SuppressWarnings({ "unchecked", "rawtypes" }) SingleOutputStreamOperator<R> returnStream = new SingleOutputStreamOperator(environment, resultTransform); getExecutionEnvironment().addOperator(resultTransform); return returnStream; }
private <OUT> DataStreamSource<OUT> createFileInput(FileInputFormat<OUT> inputFormat, TypeInformation<OUT> typeInfo, String sourceName, FileProcessingMode monitoringMode, long interval) { Preconditions.checkNotNull(inputFormat, "Unspecified file input format."); Preconditions.checkNotNull(typeInfo, "Unspecified output type information."); Preconditions.checkNotNull(sourceName, "Unspecified name for the source."); Preconditions.checkNotNull(monitoringMode, "Unspecified monitoring mode."); Preconditions.checkArgument(monitoringMode.equals(FileProcessingMode.PROCESS_ONCE) || interval >= ContinuousFileMonitoringFunction.MIN_MONITORING_INTERVAL, "The path monitoring interval cannot be less than " + ContinuousFileMonitoringFunction.MIN_MONITORING_INTERVAL + " ms."); ContinuousFileMonitoringFunction<OUT> monitoringFunction = new ContinuousFileMonitoringFunction<>(inputFormat, monitoringMode, getParallelism(), interval); ContinuousFileReaderOperator<OUT> reader = new ContinuousFileReaderOperator<>(inputFormat); SingleOutputStreamOperator<OUT> source = addSource(monitoringFunction, sourceName) .transform("Split Reader: " + sourceName, typeInfo, reader); return new DataStreamSource<>(source); }
operator, outTypeInfo, environment.getParallelism());
public static void main(String[] args) throws Exception { final ParameterTool pt = ParameterTool.fromArgs(args); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); setupEnvironment(env, pt); final int numStates = pt.getInt(NUM_LIST_STATES_PER_OP.key(), NUM_LIST_STATES_PER_OP.defaultValue()); final int numPartitionsPerState = pt.getInt(NUM_PARTITIONS_PER_LIST_STATE.key(), NUM_PARTITIONS_PER_LIST_STATE.defaultValue()); Preconditions.checkState(env.getCheckpointInterval() > 0L, "Checkpointing must be enabled for this test!"); env.addSource(new SimpleEndlessSourceWithBloatedState(numStates, numPartitionsPerState)).setParallelism(env.getParallelism()) .addSink(new DiscardingSink<>()).setParallelism(1); env.execute("HeavyDeploymentStressTestProgram"); }
@Internal private <OUT> SingleOutputStreamOperator<OUT> transform( final String functionName, final TypeInformation<OUT> outTypeInfo, final TwoInputStreamOperator<IN1, IN2, OUT> operator) { // read the output type of the input Transforms to coax out errors about MissingTypeInfo inputStream1.getType(); inputStream2.getType(); TwoInputTransformation<IN1, IN2, OUT> transform = new TwoInputTransformation<>( inputStream1.getTransformation(), inputStream2.getTransformation(), functionName, operator, outTypeInfo, environment.getParallelism()); if (inputStream1 instanceof KeyedStream) { KeyedStream<IN1, ?> keyedInput1 = (KeyedStream<IN1, ?>) inputStream1; TypeInformation<?> keyType1 = keyedInput1.getKeyType(); transform.setStateKeySelectors(keyedInput1.getKeySelector(), null); transform.setStateKeyType(keyType1); } @SuppressWarnings({ "unchecked", "rawtypes" }) SingleOutputStreamOperator<OUT> returnStream = new SingleOutputStreamOperator(environment, transform); getExecutionEnvironment().addOperator(transform); return returnStream; }
try { final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); assertEquals("test setup broken", PARALLELISM, env.getParallelism());
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(parallelism - 1); env.getConfig().setMaxParallelism(env.getParallelism());
public int getParallelism() { if (parallelism == ExecutionConfig.PARALLELISM_DEFAULT) { return env.getParallelism(); } else { return parallelism; } }
public int getParallelism() { if (parallelism == ExecutionConfig.PARALLELISM_DEFAULT) { return env.getParallelism(); } else { return parallelism; } }
public DataStreamSource(StreamExecutionEnvironment environment, TypeInformation<T> outTypeInfo, StreamSource<T, ?> operator, boolean isParallel, String sourceName) { super(environment, new SourceTransformation<>(sourceName, operator, outTypeInfo, environment.getParallelism())); this.isParallel = isParallel; if (!isParallel) { setParallelism(1); } }
public DataStreamSource(StreamExecutionEnvironment environment, TypeInformation<T> outTypeInfo, StreamSource<T, ?> operator, boolean isParallel, String sourceName) { super(environment, new SourceTransformation<>(sourceName, operator, outTypeInfo, environment.getParallelism())); this.isParallel = isParallel; if (!isParallel) { setParallelism(1); } }
private void calculateBoundedStreamScan(BatchExecBoundedStreamScan boundedStreamScan) { StreamTransformation transformation = boundedStreamScan.getSourceTransformation(tEnv.execEnv()); int parallelism = transformation.getParallelism(); if (parallelism <= 0) { parallelism = tEnv.execEnv().getParallelism(); } finalParallelismNodeMap.put(boundedStreamScan, parallelism); }
private void calculateStreamScan(StreamExecDataStreamScan streamScan) { StreamTransformation transformation = streamScan.getSourceTransformation(tEnv.execEnv()); int parallelism = transformation.getParallelism(); if (parallelism <= 0) { parallelism = tEnv.execEnv().getParallelism(); } streamScan.getResource().setParallelism(parallelism); }
@Test public void shouldSetParallelismStreaming() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); options.setParallelism(42); StreamExecutionEnvironment sev = FlinkExecutionEnvironments.createStreamExecutionEnvironment( options, Collections.emptyList()); assertThat(options.getParallelism(), is(42)); assertThat(sev.getParallelism(), is(42)); }
@Test public void shouldFallbackToDefaultParallelismStreaming() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); options.setFlinkMaster("host:80"); StreamExecutionEnvironment sev = FlinkExecutionEnvironments.createStreamExecutionEnvironment( options, Collections.emptyList()); assertThat(options.getParallelism(), is(1)); assertThat(sev.getParallelism(), is(1)); }
@Test public void useDefaultParallelismFromContextStreaming() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); StreamExecutionEnvironment sev = FlinkExecutionEnvironments.createStreamExecutionEnvironment( options, Collections.emptyList()); assertThat(sev, instanceOf(LocalStreamEnvironment.class)); assertThat(options.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); assertThat(sev.getParallelism(), is(LocalStreamEnvironment.getDefaultLocalParallelism())); }
@Test public void shouldInferParallelismFromEnvironmentStreaming() throws IOException { String confDir = extractFlinkConfig(); FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); options.setRunner(TestFlinkRunner.class); options.setFlinkMaster("host:80"); StreamExecutionEnvironment sev = FlinkExecutionEnvironments.createStreamExecutionEnvironment( options, Collections.emptyList(), confDir); assertThat(options.getParallelism(), is(23)); assertThat(sev.getParallelism(), is(23)); }