@Override public TwillPreparer addLogHandler(LogHandler handler) { getDelegate().addLogHandler(handler); return this; }
@Override public TwillPreparer addLogHandler(LogHandler handler) { getDelegate().addLogHandler(handler); return this; }
/** * Adds a {@link LogHandler} to the {@link TwillPreparer} based on the configuration. */ private void addLogHandler(TwillPreparer twillPreparer, CConfiguration cConf) { String confLevel = cConf.get(Constants.COLLECT_APP_CONTAINER_LOG_LEVEL).toUpperCase(); if ("OFF".equals(confLevel)) { twillPreparer.withConfiguration(Collections.singletonMap(Configs.Keys.LOG_COLLECTION_ENABLED, "false")); return; } LogEntry.Level logLevel = LogEntry.Level.ERROR; try { logLevel = "ALL".equals(confLevel) ? LogEntry.Level.TRACE : LogEntry.Level.valueOf(confLevel.toUpperCase()); } catch (Exception e) { LOG.warn("Invalid application container log level {}. Defaulting to ERROR.", confLevel); } twillPreparer.addLogHandler(new LoggerLogHandler(LOG, logLevel)); }
/** * Adds a {@link LogHandler} to the {@link TwillPreparer} based on the configuration. */ private void addLogHandler(TwillPreparer twillPreparer, CConfiguration cConf) { String confLevel = cConf.get(Constants.COLLECT_APP_CONTAINER_LOG_LEVEL).toUpperCase(); if ("OFF".equals(confLevel)) { twillPreparer.withConfiguration(Collections.singletonMap(Configs.Keys.LOG_COLLECTION_ENABLED, "false")); return; } LogEntry.Level logLevel = LogEntry.Level.ERROR; try { logLevel = "ALL".equals(confLevel) ? LogEntry.Level.TRACE : LogEntry.Level.valueOf(confLevel.toUpperCase()); } catch (Exception e) { LOG.warn("Invalid application container log level {}. Defaulting to ERROR.", confLevel); } twillPreparer.addLogHandler(new LoggerLogHandler(LOG, logLevel)); }
/** * Test to verify exception is thrown in case a non-existent runnable is specified in a placement policy. */ @Test(expected = IllegalArgumentException.class) public void testNonExistentRunnable() throws InterruptedException, ExecutionException, TimeoutException { TwillRunner runner = getTwillRunner(); TwillController controller = runner.prepare(new FaultyApplication()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); controller.terminate().get(120, TimeUnit.SECONDS); }
/** * Test to verify exception is thrown in case a runnable is mentioned in more than one placement policy. */ @Test(expected = IllegalArgumentException.class) public void testPlacementPolicySpecification() throws InterruptedException, ExecutionException, TimeoutException { TwillRunner runner = getTwillRunner(); TwillController controller = runner.prepare(new BadApplication()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); controller.terminate().get(120, TimeUnit.SECONDS); }
@Test public void testInitFail() throws InterruptedException, ExecutionException, TimeoutException { TwillRunner runner = getTwillRunner(); final CountDownLatch logLatch = new CountDownLatch(1); // Verify that it receives the exception log entry that thrown when runnable initialize LogHandler logVerifyHandler = new LogHandler() { @Override public void onLog(LogEntry logEntry) { if (logEntry.getMessage().endsWith("exited abnormally with state COMPLETE, exit code 10.")) { logLatch.countDown(); } } }; TwillController controller = runner .prepare(new InitFailRunnable()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))) .addLogHandler(logVerifyHandler) .start(); controller.awaitTerminated(2, TimeUnit.MINUTES); Assert.assertTrue(logLatch.await(10, TimeUnit.SECONDS)); }
@Test public void testKilled() throws IOException, InterruptedException, TimeoutException, ExecutionException { // Create a parent folder to be written by EventHandler File parentFolder = TMP_FOLDER.newFolder(); parentFolder.setWritable(true, false); TwillController controller = getTwillRunner().prepare(new SleepApplication(parentFolder.getAbsolutePath())) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); // Wait for the runnable to run and create runFile within 120 secs File runFile = new File(parentFolder, RUN_FILE); Stopwatch stopwatch = new Stopwatch().start(); while (!runFile.exists() && stopwatch.elapsedTime(TimeUnit.SECONDS) < 120) { TimeUnit.SECONDS.sleep(1); } Assert.assertTrue(runFile.exists()); // Terminate the app once the runnable runs controller.terminate(); controller.awaitTerminated(120, TimeUnit.SECONDS); // EventHandler#killed() method should be called to create a file Assert.assertTrue(new File(parentFolder.getAbsolutePath(), KILLED_FILE).exists()); // EventHandler#completed() method should not be called Assert.assertFalse(new File(parentFolder.getAbsolutePath(), COMPLETED_FILE).exists()); // EventHandler#aborted() method should not be called Assert.assertFalse(new File(parentFolder.getAbsolutePath(), ABORTED_FILE).exists()); }
@Test public void testCustomClassLoader() throws Exception { TwillController controller = getTwillRunner().prepare(new CustomClassLoaderRunnable()) .setClassLoader(CustomClassLoader.class.getName()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .setJVMOptions("-Dservice.port=54321") .setJVMOptions(CustomClassLoaderRunnable.class.getSimpleName(), "-Dservice.name=custom") .start(); Assert.assertTrue(waitForSize(controller.discoverService("custom"), 1, 120)); controller.terminate().get(); } }
@Ignore @Test public void testDistributedShell() throws InterruptedException { TwillRunner twillRunner = getTwillRunner(); TwillController controller = twillRunner.prepare(new DistributedShell("pwd", "ls -al")) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))) .start(); final CountDownLatch stopLatch = new CountDownLatch(1); controller.onTerminated(new Runnable() { @Override public void run() { stopLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(stopLatch.await(10, TimeUnit.SECONDS)); } }
@Test public void testProvisionTimeout() throws InterruptedException, ExecutionException, TimeoutException, IOException { TwillRunner runner = getTwillRunner(); // Create a parent folder to be written by EventHandler#aborted() File parentFolder = TMP_FOLDER.newFolder(); parentFolder.setWritable(true, false); TwillController controller = runner.prepare(new TimeoutApplication(parentFolder.getAbsolutePath())) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); // The provision should failed in 30 seconds after AM started, which AM could took a while to start. // Hence we give 90 seconds max time here. try { controller.awaitTerminated(90, TimeUnit.SECONDS); // EventHandler#aborted() method should be called to create a file Assert.assertTrue(new File(parentFolder.getAbsolutePath(), ABORTED_FILE).exists()); String[] abortedFiles = parentFolder.list(); Assert.assertNotNull(abortedFiles); Assert.assertEquals(1, abortedFiles.length); } finally { // If it timeout, kill the app as cleanup. controller.kill(); } }
private void maxRetriesRun(final int instances) throws TimeoutException, ExecutionException { TwillRunner runner = getTwillRunner(); final int maxRetries = 3; final AtomicInteger retriesSeen = new AtomicInteger(0); ResourceSpecification resource = ResourceSpecification.Builder.with().setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA).setInstances(instances).build(); TwillController controller = runner.prepare(new FailingServer(), resource) .withMaxRetries(FailingServer.class.getSimpleName(), maxRetries) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .addLogHandler(new LogHandler() { @Override public void onLog(LogEntry logEntry) { if (logEntry.getMessage().contains("retries for instance")) { retriesSeen.incrementAndGet(); } } }) .start(); controller.awaitTerminated(2, TimeUnit.MINUTES); Assert.assertEquals(maxRetries * instances, retriesSeen.get()); }
@Test public void testServiceDiscovery() throws InterruptedException, ExecutionException, TimeoutException { TwillRunner twillRunner = getTwillRunner(); TwillController controller = twillRunner .prepare(new ServiceApplication()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .withApplicationArguments("echo") .start(); ServiceDiscovered discovered = controller.discoverService("discovered"); Assert.assertTrue(waitForSize(discovered, 1, 120)); controller.terminate().get(); }
@Test public void testDebugPortOneRunnable() throws Exception { YarnTwillRunnerService runner = getTwillRunner(); runner.start(); TwillController controller = runner.prepare(new DummyApplication()) .enableDebugging("r1") .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))) .start(); final CountDownLatch running = new CountDownLatch(1); controller.onRunning(new Runnable() { @Override public void run() { running.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(running.await(120, TimeUnit.SECONDS)); Assert.assertTrue(waitForDebugPort(controller, "r1", 30)); controller.terminate().get(120, TimeUnit.SECONDS); // Sleep a bit before exiting. TimeUnit.SECONDS.sleep(2); }
/** * Test for requesting different container size in different order. * It specifically test for workaround for YARN-314. */ @Test public void testContainerSize() throws InterruptedException, TimeoutException, ExecutionException { TwillRunner runner = getTwillRunner(); TwillController controller = runner.prepare(new SleepApp()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); try { ServiceDiscovered discovered = controller.discoverService("sleep"); Assert.assertTrue(waitForSize(discovered, 2, 120)); } finally { controller.terminate().get(120, TimeUnit.SECONDS); } }
@Test public void testDebugPortAllRunnables() throws Exception { YarnTwillRunnerService runner = getTwillRunner(); runner.start(); TwillController controller = runner.prepare(new DummyApplication()) .enableDebugging() .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))) .start(); final CountDownLatch running = new CountDownLatch(1); controller.onRunning(new Runnable() { @Override public void run() { running.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(running.await(120, TimeUnit.SECONDS)); Assert.assertTrue(waitForDebugPort(controller, "r1", 30)); Assert.assertTrue(waitForDebugPort(controller, "r2", 30)); controller.terminate().get(120, TimeUnit.SECONDS); // Sleep a bit before exiting. TimeUnit.SECONDS.sleep(2); } }
@Test public void testRestartSingleRunnable() throws Exception { YarnTwillRunnerService runner = getTwillRunner(); runner.start(); LOG.info("Starting application {}", SingleRunnableApp.class.getSimpleName()); TwillController controller = runner.prepare(new SingleRunnableApp()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))) .start(); // Lets wait until all runnables have started waitForInstance(controller, HANGING_RUNNABLE, "002", 120, TimeUnit.SECONDS); waitForContainers(controller, 2, 60, TimeUnit.SECONDS); // Now restart runnable LOG.info("Restarting runnable {}", HANGING_RUNNABLE); controller.restartAllInstances(HANGING_RUNNABLE); waitForInstance(controller, HANGING_RUNNABLE, "003", 120, TimeUnit.SECONDS); waitForContainers(controller, 2, 60, TimeUnit.SECONDS); // Send command to HANGING_RUNNABLE to hang when stopped controller.sendCommand(HANGING_RUNNABLE, new SleepCommand(1000)).get(); LOG.info("Restarting runnable {}", HANGING_RUNNABLE); controller.restartAllInstances(HANGING_RUNNABLE); waitForInstance(controller, HANGING_RUNNABLE, "004", 120, TimeUnit.SECONDS); waitForContainers(controller, 2, 60, TimeUnit.SECONDS); }
@Test public void testExtraOptions() throws InterruptedException, ExecutionException { // Start the testing app with jvm options at both global level as well as for the specific runnables. TwillController controller = getTwillRunner() .prepare(new JvmOptionsApplication()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))) .setJVMOptions("-Dservice.name=default") .setJVMOptions("r2", "-Dservice.name=r2") .start(); // For r1 and r3 will be using "default" as the service name. waitForSize(controller.discoverService("default"), 2, 120); // r2 will be use "r2" as the service name. waitForSize(controller.discoverService("r2"), 1, 120); controller.terminate().get(); }
@Test public void testTaskCompleted() throws InterruptedException, TimeoutException, ExecutionException { TwillRunner twillRunner = getTwillRunner(); TwillController controller = twillRunner.prepare(new SleepTask(), ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA) .setInstances(3).build()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); final CountDownLatch runLatch = new CountDownLatch(1); controller.onRunning(new Runnable() { @Override public void run() { runLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(runLatch.await(1, TimeUnit.MINUTES)); controller.awaitTerminated(1, TimeUnit.MINUTES); Assert.assertEquals(ServiceController.TerminationStatus.SUCCEEDED, controller.getTerminationStatus()); }
@Test public void testFailureComplete() throws TimeoutException, ExecutionException, InterruptedException { TwillRunner twillRunner = getTwillRunner(); // Start the app with an invalid ClassLoader. This will cause the AM fails to start. TwillController controller = twillRunner.prepare(new SleepTask(), ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA) .setInstances(1).build()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .setClassLoader("InvalidClassLoader") .start(); final CountDownLatch terminateLatch = new CountDownLatch(1); controller.onTerminated(new Runnable() { @Override public void run() { terminateLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(terminateLatch.await(2, TimeUnit.MINUTES)); Assert.assertEquals(ServiceController.TerminationStatus.FAILED, controller.getTerminationStatus()); } }