@Override public void onRunning(Runnable runnable, Executor executor) { delegate.onRunning(runnable, executor); }
@Override public void onRunning(Runnable runnable, Executor executor) { delegate.onRunning(runnable, executor); }
/** * Adds a listener to the given TwillController to delete local temp files when the program has started/terminated. * The local temp files could be removed once the program is started, since Twill would keep the files in * HDFS and no long needs the local temp files once program is started. * * @return The same TwillController instance. */ private TwillController addCleanupListener(TwillController controller, final Program program, final File tempDir) { final AtomicBoolean deleted = new AtomicBoolean(false); Runnable cleanup = () -> { if (!deleted.compareAndSet(false, true)) { return; } LOG.debug("Cleanup tmp files for {}: {}", program.getId(), tempDir); deleteDirectory(tempDir); }; controller.onRunning(cleanup, Threads.SAME_THREAD_EXECUTOR); controller.onTerminated(cleanup, Threads.SAME_THREAD_EXECUTOR); return controller; }
twillController.onRunning(new Runnable() { @Override public void run() {
twillController.onRunning(new Runnable() { @Override public void run() {
/** * Adds a listener to the given TwillController to delete local temp files when the program has started/terminated. * The local temp files could be removed once the program is started, since Twill would keep the files in * HDFS and no long needs the local temp files once program is started. * * @return The same TwillController instance. */ private TwillController addCleanupListener(TwillController controller, final Program program, final File tempDir) { final AtomicBoolean deleted = new AtomicBoolean(false); Runnable cleanup = () -> { if (!deleted.compareAndSet(false, true)) { return; } LOG.debug("Cleanup tmp files for {}: {}", program.getId(), tempDir); deleteDirectory(tempDir); }; controller.onRunning(cleanup, Threads.SAME_THREAD_EXECUTOR); controller.onTerminated(cleanup, Threads.SAME_THREAD_EXECUTOR); return controller; }
controller.onRunning(cleanup, Threads.SAME_THREAD_EXECUTOR); controller.onTerminated(cleanup, Threads.SAME_THREAD_EXECUTOR); return controller;
controller.onRunning(new Runnable() { @Override public void run() {
controller.onRunning(new Runnable() { @Override public void run() {
}).when(twillController).onRunning(any(Runnable.class), any(Executor.class));
controller.onRunning(new Runnable() { @Override public void run() {
private ClusterEnriched startClusterAsync(Cluster cluster) throws YarnProvisioningHandlingException { YarnConfiguration yarnConfiguration = new YarnConfiguration(); updateYarnConfiguration(cluster, yarnConfiguration); List<Property> props = cluster.getClusterConfig().getSubPropertyList(); // only to show those props on UI/API defaultsConfigurator.getDistroTypeDefaultsConfigurator( cluster.getClusterConfig().getDistroType(), cluster.getClusterConfig().getIsSecure()).mergeProperties(props); List<Property> cleansedProperties = new ArrayList<>(); for (Property prop : props) { if (!EXCLUDED.contains(prop.getKey())) { cleansedProperties.add(prop); } } // async call - unfortunately I can not add event handlers before start of TwillController // which means we can miss failures TwillController twillController = yarnController.startCluster(yarnConfiguration, cleansedProperties); String runId = twillController.getRunId().getId(); RunId dRunId = new RunId(runId); cluster.setState(ClusterState.STARTING); cluster.setRunId(dRunId); OnRunningRunnable onRunning = new OnRunningRunnable(cluster); twillController.onRunning(onRunning, Threads.SAME_THREAD_EXECUTOR); initOnTerminatingThread(cluster, twillController); return getClusterInfo(cluster); }
}).when(twillController).onRunning(any(Runnable.class), any(Executor.class));
@Test public void testDebugPortOneRunnable() throws Exception { YarnTwillRunnerService runner = getTwillRunner(); runner.start(); TwillController controller = runner.prepare(new DummyApplication()) .enableDebugging("r1") .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))) .start(); final CountDownLatch running = new CountDownLatch(1); controller.onRunning(new Runnable() { @Override public void run() { running.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(running.await(120, TimeUnit.SECONDS)); Assert.assertTrue(waitForDebugPort(controller, "r1", 30)); controller.terminate().get(120, TimeUnit.SECONDS); // Sleep a bit before exiting. TimeUnit.SECONDS.sleep(2); }
}).when(twillController).onRunning(any(Runnable.class), any(Executor.class));
@Test public void testDebugPortAllRunnables() throws Exception { YarnTwillRunnerService runner = getTwillRunner(); runner.start(); TwillController controller = runner.prepare(new DummyApplication()) .enableDebugging() .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out))) .start(); final CountDownLatch running = new CountDownLatch(1); controller.onRunning(new Runnable() { @Override public void run() { running.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(running.await(120, TimeUnit.SECONDS)); Assert.assertTrue(waitForDebugPort(controller, "r1", 30)); Assert.assertTrue(waitForDebugPort(controller, "r2", 30)); controller.terminate().get(120, TimeUnit.SECONDS); // Sleep a bit before exiting. TimeUnit.SECONDS.sleep(2); } }
controller.onRunning(new Runnable() { @Override public void run() {
@Test public void testAppSessionExpire() throws InterruptedException, ExecutionException, TimeoutException { TwillRunner runner = getTwillRunner(); TwillController controller = runner.prepare(new SleepRunnable(600)) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); final CountDownLatch runLatch = new CountDownLatch(1); controller.onRunning(new Runnable() { @Override public void run() { runLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); // Wait for application running Assert.assertTrue(runLatch.await(60, TimeUnit.SECONDS)); // Find the app master ZK session and expire it two times, 10 seconds apart. for (int i = 0; i < 2; i++) { Assert.assertTrue(expireAppMasterZKSession(controller, 10, TimeUnit.SECONDS)); try { controller.awaitTerminated(10, TimeUnit.SECONDS); Assert.fail("Unexpected application termination."); } catch (TimeoutException e) { // OK, expected. } } controller.terminate().get(120, TimeUnit.SECONDS); }
@Test public void testTaskCompleted() throws InterruptedException, TimeoutException, ExecutionException { TwillRunner twillRunner = getTwillRunner(); TwillController controller = twillRunner.prepare(new SleepTask(), ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA) .setInstances(3).build()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); final CountDownLatch runLatch = new CountDownLatch(1); controller.onRunning(new Runnable() { @Override public void run() { runLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(runLatch.await(1, TimeUnit.MINUTES)); controller.awaitTerminated(1, TimeUnit.MINUTES); Assert.assertEquals(ServiceController.TerminationStatus.SUCCEEDED, controller.getTerminationStatus()); }
controller.onRunning(new Runnable() { @Override public void run() {