/** * Creates a {@link ResourceSpecification} based on the given configuration keys. */ private ResourceSpecification createResourceSpecification(String vCoresKey, String memoryKey, String instancesKey) { int vCores = cConf.getInt(vCoresKey); int memory = cConf.getInt(memoryKey); int instances = instanceCountMap.get(instancesKey); Preconditions.checkArgument(vCores > 0, "Number of virtual cores must be > 0 for property %s", vCoresKey); Preconditions.checkArgument(memory > 0, "Memory size must be > 0 for property %s", memoryKey); Preconditions.checkArgument(instances > 0, "Number of instances must be > 0 for property %s", instancesKey); return ResourceSpecification.Builder.with() .setVirtualCores(vCores) .setMemory(memory, ResourceSpecification.SizeUnit.MEGA) .setInstances(instances) .build(); }
.withRunnable() .add(YARN_RUNNABLE_NAME, new AppBundleRunnable(), ResourceSpecification.Builder.with() .setVirtualCores(cpuInt) .setMemory(memoryOnHeapInt+memoryOffHeapInt, ResourceSpecification.SizeUnit.MEGA)
FlowletDefinition flowletDefinition = entry.getValue(); FlowletSpecification flowletSpec = flowletDefinition.getFlowletSpec(); ResourceSpecification resourceSpec = ResourceSpecification.Builder.with() .setVirtualCores(flowletSpec.getResources().getVirtualCores()) .setMemory(flowletSpec.getResources().getMemoryMB(), ResourceSpecification.SizeUnit.MEGA)
ResourceSpecification.Builder.with().setVirtualCores(oracleNumCores) .setMemory(oracleMaxMemory, SizeUnit.MEGA).setInstances(oracleInstances).build(); ResourceSpecification.Builder.with().setVirtualCores(workerNumCores) .setMemory(workerMaxMemory, SizeUnit.MEGA).setInstances(workerInstances).build();
ResourceSpecification.Builder.with().setVirtualCores(oracleNumCores) .setMemory(oracleMaxMemory, SizeUnit.MEGA).setInstances(oracleInstances).build(); ResourceSpecification.Builder.with().setVirtualCores(workerNumCores) .setMemory(workerMaxMemory, SizeUnit.MEGA).setInstances(workerInstances).build();
TwillRunner runner = getTwillRunner(); ResourceSpecification resourceSpec = ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(2048, ResourceSpecification.SizeUnit.MEGA)
resource = ResourceSpecification.Builder.with() .setVirtualCores(RUNNABLE_CORES) .setMemory(RUNNABLE_MEMORY, ResourceSpecification.SizeUnit.MEGA) .build(); twoInstancesResource = ResourceSpecification.Builder.with() .setVirtualCores(RUNNABLE_CORES) .setMemory(RUNNABLE_MEMORY, ResourceSpecification.SizeUnit.MEGA)
ResourceSpecification resource = ResourceSpecification.Builder.with().setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA).setInstances(2).build();
ResourceSpecification.Builder.with().setVirtualCores(config.getOracleNumCores()) .setMemory(config.getOracleMaxMemory(), SizeUnit.MEGA) .setInstances(config.getOracleInstances()).build(); ResourceSpecification.Builder.with().setVirtualCores(config.getWorkerNumCores()) .setMemory(config.getWorkerMaxMemory(), SizeUnit.MEGA) .setInstances(config.getWorkerInstances()).build();
@Test public void testFailureRestart() throws Exception { TwillRunner runner = getTwillRunner(); ResourceSpecification resource = ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA) .setInstances(2) .build(); TwillController controller = runner.prepare(new FailureRunnable(), resource) .withApplicationArguments("failure") .withArguments(FailureRunnable.class.getSimpleName(), "failure2") .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); Iterable<Discoverable> discoverables = controller.discoverService("failure"); Assert.assertTrue(waitForSize(discoverables, 2, 120)); // Make sure we see the right instance IDs Assert.assertEquals(Sets.newHashSet(0, 1), getInstances(discoverables)); // Kill server with instanceId = 0 controller.sendCommand(FailureRunnable.class.getSimpleName(), Command.Builder.of("kill0").build()); // Make sure the runnable is killed. Assert.assertTrue(waitForSize(discoverables, 1, 120)); // Wait for the restart Assert.assertTrue(waitForSize(discoverables, 2, 120)); // Make sure we see the right instance IDs Assert.assertEquals(Sets.newHashSet(0, 1), getInstances(discoverables)); controller.terminate().get(120, TimeUnit.SECONDS); }
@Test public void testFailureComplete() throws TimeoutException, ExecutionException, InterruptedException { TwillRunner twillRunner = getTwillRunner(); // Start the app with an invalid ClassLoader. This will cause the AM fails to start. TwillController controller = twillRunner.prepare(new SleepTask(), ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA) .setInstances(1).build()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .setClassLoader("InvalidClassLoader") .start(); final CountDownLatch terminateLatch = new CountDownLatch(1); controller.onTerminated(new Runnable() { @Override public void run() { terminateLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(terminateLatch.await(2, TimeUnit.MINUTES)); Assert.assertEquals(ServiceController.TerminationStatus.FAILED, controller.getTerminationStatus()); } }
@Test public void testTaskCompleted() throws InterruptedException, TimeoutException, ExecutionException { TwillRunner twillRunner = getTwillRunner(); TwillController controller = twillRunner.prepare(new SleepTask(), ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA) .setInstances(3).build()) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .start(); final CountDownLatch runLatch = new CountDownLatch(1); controller.onRunning(new Runnable() { @Override public void run() { runLatch.countDown(); } }, Threads.SAME_THREAD_EXECUTOR); Assert.assertTrue(runLatch.await(1, TimeUnit.MINUTES)); controller.awaitTerminated(1, TimeUnit.MINUTES); Assert.assertEquals(ServiceController.TerminationStatus.SUCCEEDED, controller.getTerminationStatus()); }
@Override public TwillSpecification configure() { ResourceSpecification largeRes = ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(1024, ResourceSpecification.SizeUnit.MEGA) .build(); ResourceSpecification smallRes = ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA) .build(); return TwillSpecification.Builder.with() .setName("SleepApp") .withRunnable() .add("sleep1", new SleepRunnable(12345), largeRes).noLocalFiles() .add("sleep2", new SleepRunnable(12346), smallRes).noLocalFiles() .withOrder() .begin("sleep1") .nextWhenStarted("sleep2") .build(); } }
private void maxRetriesRun(final int instances) throws TimeoutException, ExecutionException { TwillRunner runner = getTwillRunner(); final int maxRetries = 3; final AtomicInteger retriesSeen = new AtomicInteger(0); ResourceSpecification resource = ResourceSpecification.Builder.with().setVirtualCores(1) .setMemory(512, ResourceSpecification.SizeUnit.MEGA).setInstances(instances).build(); TwillController controller = runner.prepare(new FailingServer(), resource) .withMaxRetries(FailingServer.class.getSimpleName(), maxRetries) .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true))) .addLogHandler(new LogHandler() { @Override public void onLog(LogEntry logEntry) { if (logEntry.getMessage().contains("retries for instance")) { retriesSeen.incrementAndGet(); } } }) .start(); controller.awaitTerminated(2, TimeUnit.MINUTES); Assert.assertEquals(maxRetries * instances, retriesSeen.get()); }
@Override public TwillSpecification configure() { return TwillSpecification.Builder.with() .setName(APPLICATION_NAME) .withRunnable() .add(PeriodicNotificationTwillRunnable.TWILL_RUNNABLE_NAME, new PeriodicNotificationTwillRunnable(), ResourceSpecification.Builder.with() .setVirtualCores(2) .setMemory(2, SizeUnit.GIGA) .setInstances(1) .build()) .withLocalFiles() .add(PeriodicNotificationTwillRunnable.CONFIG_FILE_NAME, configFile) .apply() .anyOrder() .build(); }
@Override public TwillSpecification configure() { return TwillSpecification.Builder.with() .setName("ResourceApplication") .withRunnable() .add("echo1", new EchoServer(), ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(256, ResourceSpecification.SizeUnit.MEGA) .setInstances(2).build()).noLocalFiles() .add("echo2", new EchoServer(), ResourceSpecification.Builder.with() .setVirtualCores(2) .setMemory(512, ResourceSpecification.SizeUnit.MEGA) .setInstances(1).build()).noLocalFiles() .anyOrder() .build(); } }
@Override public TwillSpecification configure() { // Make the runnable request for container smaller than 128MB (the allocation minimum) ResourceSpecification res = ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(16, ResourceSpecification.SizeUnit.MEGA) .build(); return TwillSpecification.Builder.with() .setName("MaxHeapApp") .withRunnable() .add("sleep", new MaxHeapRunnable(12345), res).noLocalFiles() .add("sleep2", new MaxHeapRunnable(23456), res).noLocalFiles() .anyOrder() .build(); } }
@Override public TwillSpecification configure() { return TwillSpecification.Builder.with() .setName("TimeoutApplication") .withRunnable() .add(new TimeoutRunnable(), ResourceSpecification.Builder.with() .setVirtualCores(1) .setMemory(8, ResourceSpecification.SizeUnit.GIGA).build()) .noLocalFiles() .anyOrder() .withEventHandler(new Handler(parentFolderPath)) .build(); } }
/** * Returns a {@link ResourceSpecification} created from the given {@link Resources} and number of instances. */ private ResourceSpecification createResourceSpec(Resources resources, int instances) { return ResourceSpecification.Builder.with() .setVirtualCores(resources.getVirtualCores()) .setMemory(resources.getMemoryMB(), ResourceSpecification.SizeUnit.MEGA) .setInstances(instances) .build(); } }
/** * Returns a {@link ResourceSpecification} created from the given {@link Resources} and number of instances. */ private ResourceSpecification createResourceSpec(Resources resources, int instances) { return ResourceSpecification.Builder.with() .setVirtualCores(resources.getVirtualCores()) .setMemory(resources.getMemoryMB(), ResourceSpecification.SizeUnit.MEGA) .setInstances(instances) .build(); } }