String.valueOf(LauncherServer.getServerInstance().getPort())); pb.environment().put(LauncherProtocol.ENV_LAUNCHER_SECRET, handle.getSecret()); try {
pb.environment().put(LauncherProtocol.ENV_LAUNCHER_PORT, String.valueOf(server.getPort())); pb.environment().put(LauncherProtocol.ENV_LAUNCHER_SECRET, secret); try {
pb.environment().put(LauncherProtocol.ENV_LAUNCHER_PORT, String.valueOf(server.getPort())); pb.environment().put(LauncherProtocol.ENV_LAUNCHER_SECRET, secret); try {
String.valueOf(LauncherServer.getServerInstance().getPort())); pb.environment().put(LauncherProtocol.ENV_LAUNCHER_SECRET, handle.getSecret()); try {
/** * Starts a Spark application. * * @see AbstractLauncher#startApplication(SparkAppHandle.Listener...) * @param listeners Listeners to add to the handle before the app is launched. * @return A handle for the launched application. */ @Override public SparkAppHandle startApplication(SparkAppHandle.Listener... listeners) throws IOException { if (builder.isClientMode(builder.getEffectiveConfig())) { LOG.warning("It's not recommended to run client-mode applications using InProcessLauncher."); } Method main = findSparkSubmit(); LauncherServer server = LauncherServer.getOrCreateServer(); InProcessAppHandle handle = new InProcessAppHandle(server); for (SparkAppHandle.Listener l : listeners) { handle.addListener(l); } String secret = server.registerHandle(handle); setConf(LauncherProtocol.CONF_LAUNCHER_PORT, String.valueOf(server.getPort())); setConf(LauncherProtocol.CONF_LAUNCHER_SECRET, secret); List<String> sparkArgs = builder.buildSparkSubmitArgs(); String[] argv = sparkArgs.toArray(new String[sparkArgs.size()]); String appName = CommandBuilderUtils.firstNonEmpty(builder.appName, builder.mainClass, "<unknown>"); handle.start(appName, main, argv); return handle; }
/** * Starts a Spark application. * * @see AbstractLauncher#startApplication(SparkAppHandle.Listener...) * @param listeners Listeners to add to the handle before the app is launched. * @return A handle for the launched application. */ @Override public SparkAppHandle startApplication(SparkAppHandle.Listener... listeners) throws IOException { if (builder.isClientMode(builder.getEffectiveConfig())) { LOG.warning("It's not recommended to run client-mode applications using InProcessLauncher."); } Method main = findSparkSubmit(); LauncherServer server = LauncherServer.getOrCreateServer(); InProcessAppHandle handle = new InProcessAppHandle(server); for (SparkAppHandle.Listener l : listeners) { handle.addListener(l); } String secret = server.registerHandle(handle); setConf(LauncherProtocol.CONF_LAUNCHER_PORT, String.valueOf(server.getPort())); setConf(LauncherProtocol.CONF_LAUNCHER_SECRET, secret); List<String> sparkArgs = builder.buildSparkSubmitArgs(); String[] argv = sparkArgs.toArray(new String[sparkArgs.size()]); String appName = CommandBuilderUtils.firstNonEmpty(builder.appName, builder.mainClass, "<unknown>"); handle.start(appName, main, argv); return handle; }
@Test public void testTimeout() throws Exception { ChildProcAppHandle handle = null; TestClient client = null; try { // LauncherServer will immediately close the server-side socket when the timeout is set // to 0. SparkLauncher.setConfig(SparkLauncher.CHILD_CONNECTION_TIMEOUT, "0"); handle = LauncherServer.newAppHandle(); Socket s = new Socket(InetAddress.getLoopbackAddress(), LauncherServer.getServerInstance().getPort()); client = new TestClient(s); waitForError(client, handle.getSecret()); } finally { SparkLauncher.launcherConfig.remove(SparkLauncher.CHILD_CONNECTION_TIMEOUT); kill(handle); close(client); } }
@Test public void testTimeout() throws Exception { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); String secret = server.registerHandle(handle); TestClient client = null; try { // LauncherServer will immediately close the server-side socket when the timeout is set // to 0. SparkLauncher.setConfig(SparkLauncher.CHILD_CONNECTION_TIMEOUT, "0"); Socket s = new Socket(InetAddress.getLoopbackAddress(), server.getPort()); client = new TestClient(s); waitForError(client, secret); } finally { SparkLauncher.launcherConfig.remove(SparkLauncher.CHILD_CONNECTION_TIMEOUT); handle.kill(); close(client); } }
@Test public void testStreamFiltering() throws Exception { ChildProcAppHandle handle = LauncherServer.newAppHandle(); TestClient client = null; try { Socket s = new Socket(InetAddress.getLoopbackAddress(), LauncherServer.getServerInstance().getPort()); client = new TestClient(s); try { client.send(new EvilPayload()); } catch (SocketException se) { // SPARK-21522: this can happen if the server closes the socket before the full message has // been written, so it's expected. It may cause false positives though (socket errors // happening for other reasons). } waitForError(client, handle.getSecret()); assertEquals(0, EvilPayload.EVIL_BIT); } finally { kill(handle); close(client); client.clientThread.join(); } }
@Test public void testStreamFiltering() throws Exception { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); String secret = server.registerHandle(handle); TestClient client = null; try { Socket s = new Socket(InetAddress.getLoopbackAddress(), server.getPort()); client = new TestClient(s); try { client.send(new EvilPayload()); } catch (SocketException se) { // SPARK-21522: this can happen if the server closes the socket before the full message has // been written, so it's expected. It may cause false positives though (socket errors // happening for other reasons). } waitForError(client, secret); assertEquals(0, EvilPayload.EVIL_BIT); } finally { handle.kill(); close(client); client.clientThread.join(); } }
@Test public void testAppHandleDisconnect() throws Exception { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); String secret = server.registerHandle(handle); TestClient client = null; try { Socket s = new Socket(InetAddress.getLoopbackAddress(), server.getPort()); client = new TestClient(s); client.send(new Hello(secret, "1.4.0")); client.send(new SetAppId("someId")); // Wait until we know the server has received the messages and matched the handle to the // connection before disconnecting. eventually(Duration.ofSeconds(1), Duration.ofMillis(10), () -> { assertEquals("someId", handle.getAppId()); }); handle.disconnect(); waitForError(client, secret); } finally { handle.kill(); close(client); client.clientThread.join(); } }
final Semaphore semaphore = new Semaphore(0); try { Socket s = new Socket(InetAddress.getLoopbackAddress(), server.getPort()); handle.addListener(new SparkAppHandle.Listener() { public void stateChanged(SparkAppHandle handle) {
@Test public void testSparkSubmitVmShutsDown() throws Exception { ChildProcAppHandle handle = LauncherServer.newAppHandle(); TestClient client = null; final Semaphore semaphore = new Semaphore(0); try { Socket s = new Socket(InetAddress.getLoopbackAddress(), LauncherServer.getServerInstance().getPort()); handle.addListener(new SparkAppHandle.Listener() { public void stateChanged(SparkAppHandle handle) { semaphore.release(); } public void infoChanged(SparkAppHandle handle) { semaphore.release(); } }); client = new TestClient(s); client.send(new Hello(handle.getSecret(), "1.4.0")); assertTrue(semaphore.tryAcquire(30, TimeUnit.SECONDS)); // Make sure the server matched the client to the handle. assertNotNull(handle.getConnection()); close(client); assertTrue(semaphore.tryAcquire(30, TimeUnit.SECONDS)); assertEquals(SparkAppHandle.State.LOST, handle.getState()); } finally { kill(handle); close(client); client.clientThread.join(); } }
try { Socket s = new Socket(InetAddress.getLoopbackAddress(), LauncherServer.getServerInstance().getPort());
Socket s = new Socket(InetAddress.getLoopbackAddress(), server.getPort());