LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); for (SparkAppHandle.Listener l : listeners) {
LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); for (SparkAppHandle.Listener l : listeners) {
@Test public void testLauncherServerReuse() throws Exception { LauncherServer server1 = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server1); handle.kill(); LauncherServer server2 = LauncherServer.getOrCreateServer(); try { assertNotSame(server1, server2); } finally { server2.unref(); } }
/** * Starts a Spark application. * * @see AbstractLauncher#startApplication(SparkAppHandle.Listener...) * @param listeners Listeners to add to the handle before the app is launched. * @return A handle for the launched application. */ @Override public SparkAppHandle startApplication(SparkAppHandle.Listener... listeners) throws IOException { if (builder.isClientMode(builder.getEffectiveConfig())) { LOG.warning("It's not recommended to run client-mode applications using InProcessLauncher."); } Method main = findSparkSubmit(); LauncherServer server = LauncherServer.getOrCreateServer(); InProcessAppHandle handle = new InProcessAppHandle(server); for (SparkAppHandle.Listener l : listeners) { handle.addListener(l); } String secret = server.registerHandle(handle); setConf(LauncherProtocol.CONF_LAUNCHER_PORT, String.valueOf(server.getPort())); setConf(LauncherProtocol.CONF_LAUNCHER_SECRET, secret); List<String> sparkArgs = builder.buildSparkSubmitArgs(); String[] argv = sparkArgs.toArray(new String[sparkArgs.size()]); String appName = CommandBuilderUtils.firstNonEmpty(builder.appName, builder.mainClass, "<unknown>"); handle.start(appName, main, argv); return handle; }
/** * Starts a Spark application. * * @see AbstractLauncher#startApplication(SparkAppHandle.Listener...) * @param listeners Listeners to add to the handle before the app is launched. * @return A handle for the launched application. */ @Override public SparkAppHandle startApplication(SparkAppHandle.Listener... listeners) throws IOException { if (builder.isClientMode(builder.getEffectiveConfig())) { LOG.warning("It's not recommended to run client-mode applications using InProcessLauncher."); } Method main = findSparkSubmit(); LauncherServer server = LauncherServer.getOrCreateServer(); InProcessAppHandle handle = new InProcessAppHandle(server); for (SparkAppHandle.Listener l : listeners) { handle.addListener(l); } String secret = server.registerHandle(handle); setConf(LauncherProtocol.CONF_LAUNCHER_PORT, String.valueOf(server.getPort())); setConf(LauncherProtocol.CONF_LAUNCHER_SECRET, secret); List<String> sparkArgs = builder.buildSparkSubmitArgs(); String[] argv = sparkArgs.toArray(new String[sparkArgs.size()]); String appName = CommandBuilderUtils.firstNonEmpty(builder.appName, builder.mainClass, "<unknown>"); handle.start(appName, main, argv); return handle; }
@Test public void testTimeout() throws Exception { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); String secret = server.registerHandle(handle); TestClient client = null; try { // LauncherServer will immediately close the server-side socket when the timeout is set // to 0. SparkLauncher.setConfig(SparkLauncher.CHILD_CONNECTION_TIMEOUT, "0"); Socket s = new Socket(InetAddress.getLoopbackAddress(), server.getPort()); client = new TestClient(s); waitForError(client, secret); } finally { SparkLauncher.launcherConfig.remove(SparkLauncher.CHILD_CONNECTION_TIMEOUT); handle.kill(); close(client); } }
@Test public void testAppHandleDisconnect() throws Exception { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); String secret = server.registerHandle(handle); TestClient client = null; try { Socket s = new Socket(InetAddress.getLoopbackAddress(), server.getPort()); client = new TestClient(s); client.send(new Hello(secret, "1.4.0")); client.send(new SetAppId("someId")); // Wait until we know the server has received the messages and matched the handle to the // connection before disconnecting. eventually(Duration.ofSeconds(1), Duration.ofMillis(10), () -> { assertEquals("someId", handle.getAppId()); }); handle.disconnect(); waitForError(client, secret); } finally { handle.kill(); close(client); client.clientThread.join(); } }
@Test public void testStreamFiltering() throws Exception { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); String secret = server.registerHandle(handle); TestClient client = null; try { Socket s = new Socket(InetAddress.getLoopbackAddress(), server.getPort()); client = new TestClient(s); try { client.send(new EvilPayload()); } catch (SocketException se) { // SPARK-21522: this can happen if the server closes the socket before the full message has // been written, so it's expected. It may cause false positives though (socket errors // happening for other reasons). } waitForError(client, secret); assertEquals(0, EvilPayload.EVIL_BIT); } finally { handle.kill(); close(client); client.clientThread.join(); } }
@Test public void testSparkSubmitVmShutsDown() throws Exception { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); String secret = server.registerHandle(handle);
@Test public void testCommunication() throws Exception { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); String secret = server.registerHandle(handle);