@Override public void close() throws IOException { synchronized (clients) { clients.remove(this); } super.close(); if (handle != null) { if (!handle.getState().isFinal()) { LOG.log(Level.WARNING, "Lost connection to spark application."); handle.setState(SparkAppHandle.State.LOST); } handle.disconnect(); } }
/** * Creates a new app handle. The handle will wait for an incoming connection for a configurable * amount of time, and if one doesn't arrive, it will transition to an error state. */ ChildProcAppHandle newAppHandle(String secret) { ChildProcAppHandle handle = new ChildProcAppHandle(secret, this); ChildProcAppHandle existing = pending.putIfAbsent(secret, handle); CommandBuilderUtils.checkState(existing == null, "Multiple handles with the same secret."); return handle; }
if (isDisposed()) { return; State currState = getState(); setState(State.FAILED, true); dispose();
@Override public synchronized void kill() { if (!isDisposed()) { setState(State.KILLED); disconnect(); if (childProc != null) { if (childProc.isAlive()) { childProc.destroyForcibly(); } childProc = null; } } }
@Test public void testSparkSubmitVmShutsDown() throws Exception { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); String secret = server.registerHandle(handle); try { Socket s = new Socket(InetAddress.getLoopbackAddress(), server.getPort()); handle.addListener(new SparkAppHandle.Listener() { public void stateChanged(SparkAppHandle handle) { semaphore.release(); assertTrue(semaphore.tryAcquire(30, TimeUnit.SECONDS)); assertNotNull(handle.getConnection()); client.close(); handle.dispose(); assertTrue(semaphore.tryAcquire(30, TimeUnit.SECONDS)); assertEquals(SparkAppHandle.State.LOST, handle.getState()); } finally { handle.kill(); close(client); client.clientThread.join();
@Test public void testCommunication() throws Exception { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); String secret = server.registerHandle(handle); handle.addListener(new SparkAppHandle.Listener() { @Override public void stateChanged(SparkAppHandle handle) { assertNotNull(handle.getConnection()); assertEquals("app-id", handle.getAppId()); assertEquals(SparkAppHandle.State.RUNNING, handle.getState()); handle.stop(); Message stopMsg = client.inbound.poll(30, TimeUnit.SECONDS); assertTrue(stopMsg instanceof Stop); } finally { close(client); handle.kill(); client.clientThread.join();
public SparkAppHandle startApplication(SparkAppHandle.Listener... listeners) throws IOException { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); for (SparkAppHandle.Listener l : listeners) { handle.addListener(l); logStream = outputToLog ? child.getInputStream() : child.getErrorStream(); handle.setChildProc(child, loggerName, logStream); } catch (IOException ioe) { handle.kill(); throw ioe;
handle.addListener(new SparkAppHandle.Listener() { @Override public void stateChanged(SparkAppHandle handle) { client.send(new Hello(handle.getSecret(), "1.4.0")); assertTrue(semaphore.tryAcquire(30, TimeUnit.SECONDS)); assertNotNull(handle.getConnection()); assertEquals("app-id", handle.getAppId()); assertEquals(SparkAppHandle.State.RUNNING, handle.getState()); handle.stop(); Message stopMsg = client.inbound.poll(30, TimeUnit.SECONDS); assertTrue(stopMsg instanceof Stop);
ChildProcAppHandle handle = LauncherServer.newAppHandle(); for (SparkAppHandle.Listener l : listeners) { handle.addListener(l); pb.environment().put(LauncherProtocol.ENV_LAUNCHER_SECRET, handle.getSecret()); try { handle.setChildProc(pb.start(), loggerName); } catch (IOException ioe) { handle.kill(); throw ioe;
@Test public void testAppHandleDisconnect() throws Exception { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); String secret = server.registerHandle(handle); TestClient client = null; try { Socket s = new Socket(InetAddress.getLoopbackAddress(), server.getPort()); client = new TestClient(s); client.send(new Hello(secret, "1.4.0")); client.send(new SetAppId("someId")); // Wait until we know the server has received the messages and matched the handle to the // connection before disconnecting. eventually(Duration.ofSeconds(1), Duration.ofMillis(10), () -> { assertEquals("someId", handle.getAppId()); }); handle.disconnect(); waitForError(client, secret); } finally { handle.kill(); close(client); client.clientThread.join(); } }
@Test public void testSparkSubmitVmShutsDown() throws Exception { ChildProcAppHandle handle = LauncherServer.newAppHandle(); TestClient client = null; final Semaphore semaphore = new Semaphore(0); try { Socket s = new Socket(InetAddress.getLoopbackAddress(), LauncherServer.getServerInstance().getPort()); handle.addListener(new SparkAppHandle.Listener() { public void stateChanged(SparkAppHandle handle) { semaphore.release(); } public void infoChanged(SparkAppHandle handle) { semaphore.release(); } }); client = new TestClient(s); client.send(new Hello(handle.getSecret(), "1.4.0")); assertTrue(semaphore.tryAcquire(30, TimeUnit.SECONDS)); // Make sure the server matched the client to the handle. assertNotNull(handle.getConnection()); close(client); assertTrue(semaphore.tryAcquire(30, TimeUnit.SECONDS)); assertEquals(SparkAppHandle.State.LOST, handle.getState()); } finally { kill(handle); close(client); client.clientThread.join(); } }
@Test public void testLauncherServerReuse() throws Exception { LauncherServer server1 = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server1); handle.kill(); LauncherServer server2 = LauncherServer.getOrCreateServer(); try { assertNotSame(server1, server2); } finally { server2.unref(); } }
ChildProcAppHandle handle = pending.remove(hello.secret); if (handle != null) { handle.setConnection(this); handle.setState(SparkAppHandle.State.CONNECTED); this.handle = handle; } else { handle.setAppId(set.appId); } else if (msg instanceof SetState) { handle.setState(((SetState)msg).state); } else { throw new IllegalArgumentException("Invalid message: " +
@Test public void testLauncherServerReuse() throws Exception { ChildProcAppHandle handle1 = null; ChildProcAppHandle handle2 = null; ChildProcAppHandle handle3 = null; try { handle1 = LauncherServer.newAppHandle(); handle2 = LauncherServer.newAppHandle(); LauncherServer server1 = handle1.getServer(); assertSame(server1, handle2.getServer()); handle1.kill(); handle2.kill(); handle3 = LauncherServer.newAppHandle(); assertNotSame(server1, handle3.getServer()); handle3.kill(); assertNull(LauncherServer.getServerInstance()); } finally { kill(handle1); kill(handle2); kill(handle3); } }
@Override public synchronized void kill() { if (!disposed) { disconnect(); } if (childProc != null) { try { childProc.exitValue(); } catch (IllegalThreadStateException e) { childProc.destroyForcibly(); } finally { childProc = null; } } }
/** * Removes the client handle from the pending list (in case it's still there), and unrefs * the server. */ void unregister(ChildProcAppHandle handle) { pending.remove(handle.getSecret()); unref(); }
public SparkAppHandle startApplication(SparkAppHandle.Listener... listeners) throws IOException { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); for (SparkAppHandle.Listener l : listeners) { handle.addListener(l); logStream = outputToLog ? child.getInputStream() : child.getErrorStream(); handle.setChildProc(child, loggerName, logStream); } catch (IOException ioe) { handle.kill(); throw ioe;
ChildProcAppHandle handle = LauncherServer.newAppHandle(); for (SparkAppHandle.Listener l : listeners) { handle.addListener(l); pb.environment().put(LauncherProtocol.ENV_LAUNCHER_SECRET, handle.getSecret()); try { handle.setChildProc(pb.start(), loggerName); } catch (IOException ioe) { handle.kill(); throw ioe;
@Test public void testTimeout() throws Exception { LauncherServer server = LauncherServer.getOrCreateServer(); ChildProcAppHandle handle = new ChildProcAppHandle(server); String secret = server.registerHandle(handle); TestClient client = null; try { // LauncherServer will immediately close the server-side socket when the timeout is set // to 0. SparkLauncher.setConfig(SparkLauncher.CHILD_CONNECTION_TIMEOUT, "0"); Socket s = new Socket(InetAddress.getLoopbackAddress(), server.getPort()); client = new TestClient(s); waitForError(client, secret); } finally { SparkLauncher.launcherConfig.remove(SparkLauncher.CHILD_CONNECTION_TIMEOUT); handle.kill(); close(client); } }
@Override public synchronized void kill() { if (!isDisposed()) { setState(State.KILLED); disconnect(); if (childProc != null) { if (childProc.isAlive()) { childProc.destroyForcibly(); } childProc = null; } } }