@Override public TwillController start(long timeout, TimeUnit timeoutUnit) { return getDelegate().start(timeout, timeoutUnit); } }
@Override public TwillController start(long timeout, TimeUnit timeoutUnit) { return getDelegate().start(timeout, timeoutUnit); } }
public TwillController startCluster(YarnConfiguration yarnConfiguration, List<Property> propertyList) { TwillController tmpController = createPreparer(yarnConfiguration, propertyList).start(); return tmpController; }
@Override public TwillController start(final long timeout, final TimeUnit timeoutUnit) { try { return impersonator.doAs(programId, () -> { // Add secure tokens if (User.isHBaseSecurityEnabled(hConf) || UserGroupInformation.isSecurityEnabled()) { addSecureStore(YarnSecureStore.create(secureStoreRenewer.createCredentials())); } return new ImpersonatedTwillController(delegate.start(timeout, timeoutUnit), impersonator, programId); }); } catch (Exception e) { throw Throwables.propagate(e); } } }
@Override public TwillController start(final long timeout, final TimeUnit timeoutUnit) { try { return impersonator.doAs(programId, () -> { // Add secure tokens if (User.isHBaseSecurityEnabled(hConf) || UserGroupInformation.isSecurityEnabled()) { addSecureStore(YarnSecureStore.create(secureStoreRenewer.createCredentials())); } return new ImpersonatedTwillController(delegate.start(timeout, timeoutUnit), impersonator, programId); }); } catch (Exception e) { throw Throwables.propagate(e); } } }
@Override public TwillController launch(TwillApplication twillApplication) { TwillPreparer twillPreparer = twillRunner .prepare(twillApplication); if (options.isDebug()) { LOG.info("Starting {} with debugging enabled.", program.getId()); twillPreparer.enableDebugging(); } TwillController twillController = twillPreparer .withDependencies(new HBaseTableUtilFactory().get().getClass()) .addSecureStore(YarnSecureStore.create(HBaseTokenUtils.obtainToken(hConf, new Credentials()))) .withApplicationArguments( String.format("--%s", RunnableOptions.JAR), copiedProgram.getJarLocation().getName(), String.format("--%s", RunnableOptions.RUNTIME_ARGS), runtimeArgs ).start(); return addCleanupListener(twillController, hConfFile, cConfFile, copiedProgram, programDir); } });
/** * Test to verify exception is thrown in case a non-existent runnable is specified in a placement policy. */ @Test(expected = IllegalArgumentException.class) public void testNonExistentRunnable() throws InterruptedException, ExecutionException, TimeoutException { TwillRunner runner = getTwillRunner(); TwillController controller = runner.prepare(new FaultyApplication()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); controller.terminate().get(120, TimeUnit.SECONDS); }
/** * Test to verify exception is thrown in case a runnable is mentioned in more than one placement policy. */ @Test(expected = IllegalArgumentException.class) public void testPlacementPolicySpecification() throws InterruptedException, ExecutionException, TimeoutException { TwillRunner runner = getTwillRunner(); TwillController controller = runner.prepare(new BadApplication()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); controller.terminate().get(120, TimeUnit.SECONDS); }
@Ignore @Test public void testDistributedShell() throws InterruptedException { TwillRunner twillRunner = getTwillRunner(); TwillController controller = twillRunner.prepare(new DistributedShell("pwd", "ls -al")) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))) .start(); final CountDownLatch stopLatch = new CountDownLatch(1); controller.onTerminated(new Runnable() { @Override public void run() { stopLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(stopLatch.await(10, TimeUnit.SECONDS)); } }
@Test public void testCustomClassLoader() throws Exception { TwillController controller = getTwillRunner().prepare(new CustomClassLoaderRunnable()) .setClassLoader(CustomClassLoader.class.getName()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .setJVMOptions("-Dservice.port=54321") .setJVMOptions(CustomClassLoaderRunnable.class.getSimpleName(), "-Dservice.name=custom") .start(); Assert.assertTrue(waitForSize(controller.discoverService("custom"), 1, 120)); controller.terminate().get(); } }
@Test public void testProvisionTimeout() throws InterruptedException, ExecutionException, TimeoutException, IOException { TwillRunner runner = getTwillRunner(); // Create a parent folder to be written by EventHandler#aborted() File parentFolder = TMP_FOLDER.newFolder(); parentFolder.setWritable(true, false); TwillController controller = runner.prepare(new TimeoutApplication(parentFolder.getAbsolutePath())) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); // The provision should failed in 30 seconds after AM started, which AM could took a while to start. // Hence we give 90 seconds max time here. try { controller.awaitTerminated(90, TimeUnit.SECONDS); // EventHandler#aborted() method should be called to create a file Assert.assertTrue(new File(parentFolder.getAbsolutePath(), ABORTED_FILE).exists()); String[] abortedFiles = parentFolder.list(); Assert.assertNotNull(abortedFiles); Assert.assertEquals(1, abortedFiles.length); } finally { // If it timeout, kill the app as cleanup. controller.kill(); } }
@Test public void testServiceDiscovery() throws InterruptedException, ExecutionException, TimeoutException { TwillRunner twillRunner = getTwillRunner(); TwillController controller = twillRunner .prepare(new ServiceApplication()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .withApplicationArguments("echo") .start(); ServiceDiscovered discovered = controller.discoverService("discovered"); Assert.assertTrue(waitForSize(discovered, 1, 120)); controller.terminate().get(); }
@Test public void testInitFail() throws InterruptedException, ExecutionException, TimeoutException { TwillRunner runner = getTwillRunner(); final CountDownLatch logLatch = new CountDownLatch(1); // Verify that it receives the exception log entry that thrown when runnable initialize LogHandler logVerifyHandler = new LogHandler() { @Override public void onLog(LogEntry logEntry) { if (logEntry.getMessage().endsWith("exited abnormally with state COMPLETE, exit code 10.")) { logLatch.countDown(); } } }; TwillController controller = runner .prepare(new InitFailRunnable()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))) .addLogHandler(logVerifyHandler) .start(); controller.awaitTerminated(2, TimeUnit.MINUTES); Assert.assertTrue(logLatch.await(10, TimeUnit.SECONDS)); }
@Test public void testDebugPortOneRunnable() throws Exception { YarnTwillRunnerService runner = getTwillRunner(); runner.start(); TwillController controller = runner.prepare(new DummyApplication()) .enableDebugging("r1") .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))) .start(); final CountDownLatch running = new CountDownLatch(1); controller.onRunning(new Runnable() { @Override public void run() { running.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(running.await(120, TimeUnit.SECONDS)); Assert.assertTrue(waitForDebugPort(controller, "r1", 30)); controller.terminate().get(120, TimeUnit.SECONDS); // Sleep a bit before exiting. TimeUnit.SECONDS.sleep(2); }
/** * Test for requesting different container size in different order. * It specifically test for workaround for YARN-314. */ @Test public void testContainerSize() throws InterruptedException, TimeoutException, ExecutionException { TwillRunner runner = getTwillRunner(); TwillController controller = runner.prepare(new SleepApp()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); try { ServiceDiscovered discovered = controller.discoverService("sleep"); Assert.assertTrue(waitForSize(discovered, 2, 120)); } finally { controller.terminate().get(120, TimeUnit.SECONDS); } }
@Test public void testDebugPortAllRunnables() throws Exception { YarnTwillRunnerService runner = getTwillRunner(); runner.start(); TwillController controller = runner.prepare(new DummyApplication()) .enableDebugging() .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))) .start(); final CountDownLatch running = new CountDownLatch(1); controller.onRunning(new Runnable() { @Override public void run() { running.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(running.await(120, TimeUnit.SECONDS)); Assert.assertTrue(waitForDebugPort(controller, "r1", 30)); Assert.assertTrue(waitForDebugPort(controller, "r2", 30)); controller.terminate().get(120, TimeUnit.SECONDS); // Sleep a bit before exiting. TimeUnit.SECONDS.sleep(2); } }
@Test public void testExtraOptions() throws InterruptedException, ExecutionException { // Start the testing app with jvm options at both global level as well as for the specific runnables. TwillController controller = getTwillRunner() .prepare(new JvmOptionsApplication()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))) .setJVMOptions("-Dservice.name=default") .setJVMOptions("r2", "-Dservice.name=r2") .start(); // For r1 and r3 will be using "default" as the service name. waitForSize(controller.discoverService("default"), 2, 120); // r2 will be use "r2" as the service name. waitForSize(controller.discoverService("r2"), 1, 120); controller.terminate().get(); }
private void maxRetriesRun(final int instances) throws TimeoutException, ExecutionException { TwillRunner runner = getTwillRunner(); final int maxRetries = 3; final AtomicInteger retriesSeen = new AtomicInteger(0); ResourceSpecification resource = ResourceSpecification.Builder.with().setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA).setInstances(instances).build(); TwillController controller = runner.prepare(new FailingServer(), resource) .withMaxRetries(FailingServer.class.getSimpleName(), maxRetries) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .addLogHandler(new LogHandler() { @Override public void onLog(LogEntry logEntry) { if (logEntry.getMessage().contains("retries for instance")) { retriesSeen.incrementAndGet(); } } }) .start(); controller.awaitTerminated(2, TimeUnit.MINUTES); Assert.assertEquals(maxRetries * instances, retriesSeen.get()); }
@Test public void testTaskCompleted() throws InterruptedException, TimeoutException, ExecutionException { TwillRunner twillRunner = getTwillRunner(); TwillController controller = twillRunner.prepare(new SleepTask(), ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA) .setInstances(3).build()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); final CountDownLatch runLatch = new CountDownLatch(1); controller.onRunning(new Runnable() { @Override public void run() { runLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(runLatch.await(1, TimeUnit.MINUTES)); controller.awaitTerminated(1, TimeUnit.MINUTES); Assert.assertEquals(ServiceController.TerminationStatus.SUCCEEDED, controller.getTerminationStatus()); }
@Test public void testFailureComplete() throws TimeoutException, ExecutionException, InterruptedException { TwillRunner twillRunner = getTwillRunner(); // Start the app with an invalid ClassLoader. This will cause the AM fails to start. TwillController controller = twillRunner.prepare(new SleepTask(), ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA) .setInstances(1).build()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .setClassLoader("InvalidClassLoader") .start(); final CountDownLatch terminateLatch = new CountDownLatch(1); controller.onTerminated(new Runnable() { @Override public void run() { terminateLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(terminateLatch.await(2, TimeUnit.MINUTES)); Assert.assertEquals(ServiceController.TerminationStatus.FAILED, controller.getTerminationStatus()); } }