@Override public void stateChanged(SparkAppHandle sparkAppHandle) { LOG.info("Spark app transitioned to state = " + sparkAppHandle.getState()); if (sparkAppHandle.getState().isFinal() || sparkAppHandle.getState().equals( SparkAppHandle.State.RUNNING)) { this.shutdownLatch.countDown(); sparkAppHandle.disconnect(); LOG.info("Successfully disconnected from Spark app handle"); } if (FAILED_SPARK_STATES.contains(sparkAppHandle.getState())) { this.rpcServer.cancelClient(this.clientId, "Spark app launcher failed," + " transitioned to state " + sparkAppHandle.getState()); } }
@Test public void testInProcessLauncherDoesNotKillJvm() throws Exception { SparkSubmitOptionParser opts = new SparkSubmitOptionParser(); List<String[]> wrongArgs = Arrays.asList( new String[] { "--unknown" }, new String[] { opts.DEPLOY_MODE, "invalid" }); for (String[] args : wrongArgs) { InProcessLauncher launcher = new InProcessLauncher() .setAppResource(SparkLauncher.NO_RESOURCE); switch (args.length) { case 2: launcher.addSparkArg(args[0], args[1]); break; case 1: launcher.addSparkArg(args[0]); break; default: fail("FIXME: invalid test."); } SparkAppHandle handle = launcher.startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.FAILED, handle.getState()); } // Run --version, which is useless as a use case, but should succeed and not exit the JVM. // The expected state is "LOST" since "--version" doesn't report state back to the handle. SparkAppHandle handle = new InProcessLauncher().addSparkArg(opts.VERSION).startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.LOST, handle.getState()); }
@Test public void testInProcessLauncherDoesNotKillJvm() throws Exception { SparkSubmitOptionParser opts = new SparkSubmitOptionParser(); List<String[]> wrongArgs = Arrays.asList( new String[] { "--unknown" }, new String[] { opts.DEPLOY_MODE, "invalid" }); for (String[] args : wrongArgs) { InProcessLauncher launcher = new InProcessLauncher() .setAppResource(SparkLauncher.NO_RESOURCE); switch (args.length) { case 2: launcher.addSparkArg(args[0], args[1]); break; case 1: launcher.addSparkArg(args[0]); break; default: fail("FIXME: invalid test."); } SparkAppHandle handle = launcher.startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.FAILED, handle.getState()); } // Run --version, which is useless as a use case, but should succeed and not exit the JVM. // The expected state is "LOST" since "--version" doesn't report state back to the handle. SparkAppHandle handle = new InProcessLauncher().addSparkArg(opts.VERSION).startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.LOST, handle.getState()); }
protected void waitFor(final SparkAppHandle handle) throws Exception { try { eventually(Duration.ofSeconds(10), Duration.ofMillis(10), () -> { assertTrue("Handle is not in final state.", handle.getState().isFinal()); }); } finally { if (!handle.getState().isFinal()) { handle.kill(); } } // Wait until the handle has been marked as disposed, to make sure all cleanup tasks // have been performed. AbstractAppHandle ahandle = (AbstractAppHandle) handle; eventually(Duration.ofSeconds(10), Duration.ofMillis(10), () -> { assertTrue("Handle is still not marked as disposed.", ahandle.isDisposed()); }); }
@Test public void testProcMonitorWithOutputRedirection() throws Exception { assumeFalse(isWindows()); File err = Files.createTempFile("out", "txt").toFile(); err.deleteOnExit(); SparkAppHandle handle = new TestSparkLauncher() .redirectError() .redirectOutput(err) .startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.LOST, handle.getState()); }
@Test public void testErrorPropagation() throws Exception { SparkAppHandle app = startTest(TEST_FAILURE); waitFor(app); assertEquals(SparkAppHandle.State.FAILED, app.getState()); assertNotNull(lastError); assertEquals(TEST_FAILURE_MESSAGE, lastError.getMessage()); }
@Test public void testProcMonitorWithLogRedirection() throws Exception { assumeFalse(isWindows()); SparkAppHandle handle = new TestSparkLauncher() .redirectToLog(getClass().getName()) .startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.LOST, handle.getState()); }
@Test public void testFailedChildProc() throws Exception { assumeFalse(isWindows()); SparkAppHandle handle = new TestSparkLauncher(1) .redirectToLog(getClass().getName()) .startApplication(); waitFor(handle); assertEquals(SparkAppHandle.State.FAILED, handle.getState()); }
@Test public void testLauncher() throws Exception { SparkAppHandle app = startTest(TEST_SUCCESS); waitFor(app); assertNull(lastError); // Because the test doesn't implement the launcher protocol, the final state here will be // LOST instead of FINISHED. assertEquals(SparkAppHandle.State.LOST, app.getState()); }
@Test public void testKill() throws Exception { SparkAppHandle app = startTest(TEST_KILL); app.kill(); waitFor(app); assertNull(lastError); assertEquals(SparkAppHandle.State.KILLED, app.getState()); }