SparkAppHandle h = (SparkAppHandle) invocation.getArguments()[0]; synchronized (transitions) { transitions.add(h.getState()); assertNotEquals(SparkAppHandle.State.UNKNOWN, _handle.getState()); }); assertEquals(SparkAppHandle.State.FINISHED, handle.getState()); } finally { if (handle != null) { handle.kill();
@VisibleForTesting static Future<Void> createSparkLauncherFuture(CountDownLatch shutdownLatch, SparkAppHandle sparkAppHandle, RpcServer rpcServer, String clientId) { // Monitor the countdown latch Callable<Void> runnable = () -> { try { shutdownLatch.await(); } catch (InterruptedException e) { rpcServer.cancelClient(clientId, "Spark app launcher interrupted"); sparkAppHandle.stop(); } return null; }; FutureTask<Void> futureTask = new FutureTask<>(runnable); Thread driverThread = new Thread(futureTask); driverThread.setDaemon(true); driverThread.setName("SparkLauncherMonitor"); driverThread.start(); return futureTask; }
@Override public void stateChanged(SparkAppHandle sparkAppHandle) { LOG.info("Spark app transitioned to state = " + sparkAppHandle.getState()); if (sparkAppHandle.getState().isFinal() || sparkAppHandle.getState().equals( SparkAppHandle.State.RUNNING)) { this.shutdownLatch.countDown(); sparkAppHandle.disconnect(); LOG.info("Successfully disconnected from Spark app handle"); } if (FAILED_SPARK_STATES.contains(sparkAppHandle.getState())) { this.rpcServer.cancelClient(this.clientId, "Spark app launcher failed," + " transitioned to state " + sparkAppHandle.getState()); } }
@Test public void testInProcessLauncherDoesNotKillJvm() throws Exception { SparkSubmitOptionParser opts = new SparkSubmitOptionParser(); List<String[]> wrongArgs = Arrays.asList( new String[] { "--unknown" }, new String[] { opts.DEPLOY_MODE, "invalid" }); for (String[] args : wrongArgs) { InProcessLauncher launcher = new InProcessLauncher() .setAppResource(SparkLauncher.NO_RESOURCE); switch (args.length) { case 2: launcher.addSparkArg(args[0], args[1]); break; case 1: launcher.addSparkArg(args[0]); break; default: fail("FIXME: invalid test."); } SparkAppHandle handle = launcher.startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.FAILED, handle.getState()); } // Run --version, which is useless as a use case, but should succeed and not exit the JVM. // The expected state is "LOST" since "--version" doesn't report state back to the handle. SparkAppHandle handle = new InProcessLauncher().addSparkArg(opts.VERSION).startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.LOST, handle.getState()); }
private void kill(SparkAppHandle handle) { if (handle != null) { handle.kill(); } }
@Test public void testInProcessLauncherDoesNotKillJvm() throws Exception { SparkSubmitOptionParser opts = new SparkSubmitOptionParser(); List<String[]> wrongArgs = Arrays.asList( new String[] { "--unknown" }, new String[] { opts.DEPLOY_MODE, "invalid" }); for (String[] args : wrongArgs) { InProcessLauncher launcher = new InProcessLauncher() .setAppResource(SparkLauncher.NO_RESOURCE); switch (args.length) { case 2: launcher.addSparkArg(args[0], args[1]); break; case 1: launcher.addSparkArg(args[0]); break; default: fail("FIXME: invalid test."); } SparkAppHandle handle = launcher.startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.FAILED, handle.getState()); } // Run --version, which is useless as a use case, but should succeed and not exit the JVM. // The expected state is "LOST" since "--version" doesn't report state back to the handle. SparkAppHandle handle = new InProcessLauncher().addSparkArg(opts.VERSION).startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.LOST, handle.getState()); }
SparkAppHandle h = (SparkAppHandle) invocation.getArguments()[0]; synchronized (transitions) { transitions.add(h.getState()); assertNotEquals(SparkAppHandle.State.UNKNOWN, _handle.getState()); }); assertEquals(SparkAppHandle.State.FINISHED, handle.getState()); } finally { if (handle != null) { handle.kill();
@Test public void testProcMonitorWithOutputRedirection() throws Exception { assumeFalse(isWindows()); File err = Files.createTempFile("out", "txt").toFile(); err.deleteOnExit(); SparkAppHandle handle = new TestSparkLauncher() .redirectError() .redirectOutput(err) .startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.LOST, handle.getState()); }
public boolean stop() { try { appHandle.stop(); //streamingContext.stop(); //streamingContext.awaitTermination(); return true; } catch (Exception e) { e.printStackTrace(); return false; } }
protected void waitFor(final SparkAppHandle handle) throws Exception { try { eventually(Duration.ofSeconds(10), Duration.ofMillis(10), () -> { assertTrue("Handle is not in final state.", handle.getState().isFinal()); }); } finally { if (!handle.getState().isFinal()) { handle.kill(); } } // Wait until the handle has been marked as disposed, to make sure all cleanup tasks // have been performed. AbstractAppHandle ahandle = (AbstractAppHandle) handle; eventually(Duration.ofSeconds(10), Duration.ofMillis(10), () -> { assertTrue("Handle is still not marked as disposed.", ahandle.isDisposed()); }); }
@Test public void testErrorPropagation() throws Exception { SparkAppHandle app = startTest(TEST_FAILURE); waitFor(app); assertEquals(SparkAppHandle.State.FAILED, app.getState()); assertNotNull(lastError); assertEquals(TEST_FAILURE_MESSAGE, lastError.getMessage()); }
@Test public void testProcMonitorWithLogRedirection() throws Exception { assumeFalse(isWindows()); SparkAppHandle handle = new TestSparkLauncher() .redirectToLog(getClass().getName()) .startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.LOST, handle.getState()); }
@Test public void testFailedChildProc() throws Exception { assumeFalse(isWindows()); SparkAppHandle handle = new TestSparkLauncher(1) .redirectToLog(getClass().getName()) .startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.FAILED, handle.getState()); }
@Test public void testLauncher() throws Exception { SparkAppHandle app = startTest(TEST_SUCCESS); waitFor(app); assertNull(lastError); // Because the test doesn't implement the launcher protocol, the final state here will be // LOST instead of FINISHED. assertEquals(SparkAppHandle.State.LOST, app.getState()); }