/** * A thin wrapper layer over {@link StreamExecutionEnvironment#createRemoteEnvironment(String, int, String...)}. * * @param host The host name or address of the master (JobManager), where the * program should be executed. * @param port The port of the master (JobManager), where the program should * be executed. * @param jar_files The JAR files with code that needs to be shipped to the * cluster. If the program uses user-defined functions, * user-defined input formats, or any libraries, those must be * provided in the JAR files. * @return A remote environment that executes the program on a cluster. */ public PythonStreamExecutionEnvironment create_remote_execution_environment( String host, int port, String... jar_files) { return new PythonStreamExecutionEnvironment( StreamExecutionEnvironment.createRemoteEnvironment(host, port, jar_files), new Path(localTmpPath), scriptName); }
/** * A thin wrapper layer over {@link StreamExecutionEnvironment#createRemoteEnvironment( *String, int, Configuration, String...)}. * * @param host The host name or address of the master (JobManager), where the * program should be executed. * @param port The port of the master (JobManager), where the program should * be executed. * @param config The configuration used by the client that connects to the remote cluster. * @param jar_files The JAR files with code that needs to be shipped to the * cluster. If the program uses user-defined functions, * user-defined input formats, or any libraries, those must be * provided in the JAR files. * @return A remote environment that executes the program on a cluster. */ public PythonStreamExecutionEnvironment create_remote_execution_environment( String host, int port, Configuration config, String... jar_files) { return new PythonStreamExecutionEnvironment( StreamExecutionEnvironment.createRemoteEnvironment(host, port, config, jar_files), new Path(localTmpPath), scriptName); }
/** * A thin wrapper layer over {@link StreamExecutionEnvironment#createRemoteEnvironment( *String, int, int, String...)}. * * @param host The host name or address of the master (JobManager), where the * program should be executed. * @param port The port of the master (JobManager), where the program should * be executed. * @param parallelism The parallelism to use during the execution. * @param jar_files The JAR files with code that needs to be shipped to the * cluster. If the program uses user-defined functions, * user-defined input formats, or any libraries, those must be * provided in the JAR files. * @return A remote environment that executes the program on a cluster. */ public PythonStreamExecutionEnvironment create_remote_execution_environment( String host, int port, int parallelism, String... jar_files) { return new PythonStreamExecutionEnvironment( StreamExecutionEnvironment.createRemoteEnvironment(host, port, parallelism, jar_files), new Path(localTmpPath), scriptName); } }
/** * Verifies that the port passed to the RemoteStreamEnvironment is used for connecting to the cluster. */ @Test public void testPortForwarding() throws Exception { String host = "fakeHost"; int port = 99; JobExecutionResult expectedResult = new JobExecutionResult(null, 0, null); RestClusterClient mockedClient = Mockito.mock(RestClusterClient.class); when(mockedClient.run(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())) .thenReturn(expectedResult); PowerMockito.whenNew(RestClusterClient.class).withAnyArguments().thenAnswer((invocation) -> { Object[] args = invocation.getArguments(); Configuration config = (Configuration) args[0]; Assert.assertEquals(host, config.getString(RestOptions.ADDRESS)); Assert.assertEquals(port, config.getInteger(RestOptions.PORT)); return mockedClient; } ); final Configuration clientConfiguration = new Configuration(); final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment( host, port, clientConfiguration); env.fromElements(1).map(x -> x * 2); JobExecutionResult actualResult = env.execute("fakeJobName"); Assert.assertEquals(expectedResult, actualResult); }
StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment( "localhost", 1337, // not needed since we use ZooKeeper
String[] parts = masterUrl.split(":"); List<String> stagingFiles = options.getFilesToStage(); this.flinkStreamEnv = StreamExecutionEnvironment.createRemoteEnvironment(parts[0], Integer.parseInt(parts[1]), stagingFiles.toArray(new String[stagingFiles.size()])); } else {
String[] parts = masterUrl.split(":"); List<String> stagingFiles = options.getFilesToStage(); flinkStreamEnv = StreamExecutionEnvironment.createRemoteEnvironment(parts[0], Integer.parseInt(parts[1]), stagingFiles.toArray(new String[stagingFiles.size()])); } else {
clientConfig.setInteger(RestOptions.PORT, Integer.parseInt(parts.get(1))); flinkStreamEnv = StreamExecutionEnvironment.createRemoteEnvironment( parts.get(0), Integer.parseInt(parts.get(1)),
@Override public void prepareRuntime() throws SpRuntimeException { if (debug) { this.env = StreamExecutionEnvironment.createLocalEnvironment(); } else { this.env = StreamExecutionEnvironment .createRemoteEnvironment(config.getHost(), config.getPort(), config.getJarFile()); } appendEnvironmentConfig(this.env); // Add the first source to the topology DataStream<Map<String, Object>> messageStream1; SourceFunction<String> source1 = getStream1Source(); if (source1 != null) { messageStream1 = env .addSource(source1).flatMap(new JsonToMapFormat()).flatMap(new StatisticLogger(getGraph())); } else { throw new SpRuntimeException("At least one source must be defined for a flink sepa"); } DataStream<Map<String, Object>> messageStream2; SourceFunction<String> source2 = getStream2Source(); if (source2 != null) { messageStream2 = env .addSource(source2).flatMap(new JsonToMapFormat()).flatMap(new StatisticLogger(getGraph())); appendExecutionConfig(messageStream1, messageStream2); } else { appendExecutionConfig(messageStream1); } }