private int getNumberOfProvisionedInstances(String flowletId) { return twillController.getResourceReport().getRunnableResources(flowletId).size(); } }
@Override public JsonElement serialize(ResourceReport src, Type typeOfSrc, JsonSerializationContext context) { JsonObject json = new JsonObject(); json.addProperty("appMasterId", src.getApplicationId()); json.add("appMasterResources", context.serialize(src.getAppMasterResources(), TwillRunResources.class)); json.add("runnableResources", context.serialize( src.getResources(), new TypeToken<Map<String, Collection<TwillRunResources>>>() { }.getType())); json.add("services", context.serialize( src.getServices(), new TypeToken<List<String>>() { }.getType())); return json; }
/** * Checks to see if the report has loaded. * @param report - The {@link ResourceReport} for this Twill Application. * @return Return true if the report is null or incomplete. Return false if the report is completely loaded. */ private boolean reportIsLoading(@Nullable final ResourceReport report) { if(report == null) { return true; } final String yarnApplicationID = report.getApplicationId(); final Collection<TwillRunResources> runnableResources = report.getResources().get(PeriodicNotificationTwillRunnable.TWILL_RUNNABLE_NAME); if(runnableResources == null || runnableResources.isEmpty()) { LOG.info("Received Resource Report for YARN ApplicationID: {}, runnable resources are still loading...", yarnApplicationID); return true; } else { LOG.info("Received Resource Report for YARN ApplicationID: {}, runnable resources are loaded.", yarnApplicationID); return false; } }
DistributedProgramLiveInfo liveInfo = new DistributedProgramLiveInfo(program, report.getApplicationId()); for (Map.Entry<String, Collection<TwillRunResources>> entry : report.getResources().entrySet()) { for (TwillRunResources resources : entry.getValue()) { liveInfo.addContainer(new ContainerInfo(containerType, liveInfo.addServices(report.getServices()); return liveInfo;
/** * Terminates all instances of the {@link PeriodicNotificationTwillApp} on the YARN cluster. */ public void stopApp() { LOG.info("Stopping any running instances..."); int counter = 0; // It is possible that we have launched multiple instances of the app. For now, stop them all, one at a time. for(final TwillController c : twillRunner.lookup(PeriodicNotificationTwillApp.APPLICATION_NAME)) { final ResourceReport report = c.getResourceReport(); LOG.info("Attempting to stop {} with YARN ApplicationId: {} and Twill RunId: {}", PeriodicNotificationTwillApp.APPLICATION_NAME, report.getApplicationId(), c.getRunId()); Futures.getUnchecked(c.terminate()); LOG.info("Stopped {} with YARN ApplicationId: {} and Twill RunId: {}", PeriodicNotificationTwillApp.APPLICATION_NAME, report.getApplicationId(), c.getRunId()); counter++; } LOG.info("Stopped {} instance(s) of {}", counter, PeriodicNotificationTwillApp.APPLICATION_NAME); }
@Override public Map<String, Integer> getFlowInfo(String flowName) { Map<String, Integer> flowletInfo = Maps.newHashMap(); try { Iterable<TwillController> controllers = lookupFlow(flowName); for (TwillController controller : controllers) { ResourceReport report = controller.getResourceReport(); sleepForZK(report); for (Map.Entry<String, Collection<TwillRunResources>> entry : report.getResources().entrySet()) { flowletInfo.put(entry.getKey(), entry.getValue().size()); } } } catch (Exception e) { LOG.warn(e.getMessage(), e); } return flowletInfo; }
Assert.assertEquals(256, resourceReport.getAppMasterResources().getMemoryMB()); Assert.assertEquals(Resources.computeMaxHeapSize(256, Configs.Defaults.YARN_AM_RESERVED_MEMORY_MB, 0.65d), resourceReport.getAppMasterResources().getMaxHeapMemoryMB()); Collection<TwillRunResources> runnableResources = resourceReport.getRunnableResources("sleep"); Assert.assertFalse(runnableResources.isEmpty()); TwillRunResources resources = runnableResources.iterator().next(); runnableResources = resourceReport.getRunnableResources("sleep2"); Assert.assertFalse(runnableResources.isEmpty()); resources = runnableResources.iterator().next();
ResourceReport report = getResourceReport(controller, 10000); Map<String, Collection<TwillRunResources>> usedResources = report.getResources(); Assert.assertEquals(2, usedResources.keySet().size()); Assert.assertTrue(usedResources.containsKey("echo1")); Assert.assertEquals(ImmutableSet.of("echo", "echo1", "echo2"), ImmutableSet.copyOf(report.getServices())); usedResources = report.getResources(); Assert.assertEquals(2, usedResources.keySet().size()); Assert.assertTrue(usedResources.containsKey("echo1")); Assert.assertEquals(report.getResources(), newReport.getResources());
@Override public Iterator<String> iterator() { return getResourceReport(controller, 10000).getServices().iterator(); } }, 3, 120);
@Override public void reportResources() { for (TwillRunner.LiveInfo info : twillRunner.lookupLive()) { Map<String, String> metricContext = getMetricContext(info); if (metricContext == null) { continue; } // will have multiple controllers if there are multiple runs of the same application for (TwillController controller : info.getControllers()) { ResourceReport report = controller.getResourceReport(); if (report == null) { continue; } int memory = report.getAppMasterResources().getMemoryMB(); int vcores = report.getAppMasterResources().getVirtualCores(); Map<String, String> runContext = ImmutableMap.<String, String>builder() .putAll(metricContext) .put(Constants.Metrics.Tag.RUN_ID, controller.getRunId().getId()).build(); sendMetrics(runContext, 1, memory, vcores); } } reportYarnResources(); }
DistributedProgramLiveInfo liveInfo = new DistributedProgramLiveInfo(program, report.getApplicationId()); Containers.ContainerType.valueOf(program.getType().name()); for (Map.Entry<String, Collection<TwillRunResources>> entry : report.getResources().entrySet()) { for (TwillRunResources resources : entry.getValue()) { liveInfo.addContainer(new ContainerInfo(containerType, liveInfo.addServices(report.getServices()); return liveInfo;
LOG.error("Unable to create ProgramController for program {} for twill application {}. It is likely caused by " + "invalid CDAP program runtime extension.", programId, resourceReport == null ? "'unknown twill application'" : resourceReport.getApplicationId()); return null;
@Override public void setInstances(String flowName, String flowletName, int instanceCount) { try { Iterable<TwillController> controllers = lookupFlow(flowName); for (TwillController controller : controllers) { ResourceReport report = controller.getResourceReport(); sleepForZK(report); int oldInstances = report.getResources().get(flowletName).size(); Program program = Programs.create(location.append(flowName)); Multimap<String, QueueName> consumerQueues = FlowUtils.configureQueue(program, program.getSpecification(), queueAdmin); DistributedFlowletInstanceUpdater instanceUpdater = new DistributedFlowletInstanceUpdater( program, controller, queueAdmin, consumerQueues); FlowTwillProgramController flowController = new FlowTwillProgramController(program.getName(), controller, instanceUpdater); Map<String, String> instanceOptions = Maps.newHashMap(); instanceOptions.put("flowlet", flowletName); instanceOptions.put("newInstances", String.valueOf(instanceCount)); instanceOptions.put("oldInstances", String.valueOf(oldInstances)); flowController.command(ProgramOptionConstants.FLOWLET_INSTANCES, instanceOptions).get(); } } catch (Exception e) { LOG.warn(e.getMessage(), e); } }
@Override public List<String> getServices(String flowName) { List<String> services = Lists.newArrayList(); try { Iterable<TwillController> controllers = lookupFlow(flowName); for (TwillController controller : controllers) { ResourceReport report = controller.getResourceReport(); sleepForZK(report); services.addAll(report.getServices()); } } catch (Exception e) { LOG.warn(e.getMessage(), e); } return services; }
@Override public void reportResources() { for (TwillRunner.LiveInfo info : twillRunner.lookupLive()) { Map<String, String> metricContext = getMetricContext(info); if (metricContext == null) { continue; } // will have multiple controllers if there are multiple runs of the same application for (TwillController controller : info.getControllers()) { ResourceReport report = controller.getResourceReport(); if (report == null) { continue; } int memory = report.getAppMasterResources().getMemoryMB(); int vcores = report.getAppMasterResources().getVirtualCores(); Map<String, String> runContext = ImmutableMap.<String, String>builder() .putAll(metricContext) .put(Constants.Metrics.Tag.RUN_ID, controller.getRunId().getId()).build(); sendMetrics(runContext, 1, memory, vcores); } } reportYarnResources(); }
private int getNumberOfProvisionedInstances(String flowletId) { return twillController.getResourceReport().getRunnableResources(flowletId).size(); }
@Override public JsonElement serialize(ResourceReport src, Type typeOfSrc, JsonSerializationContext context) { JsonObject json = new JsonObject(); json.addProperty("appMasterId", src.getApplicationId()); json.add("appMasterResources", context.serialize(src.getAppMasterResources(), TwillRunResources.class)); json.add("runnableResources", context.serialize( src.getResources(), new TypeToken<Map<String, Collection<TwillRunResources>>>() { }.getType())); json.add("services", context.serialize( src.getServices(), new TypeToken<List<String>>() { }.getType())); return json; }
LOG.error("Unable to create ProgramController for program {} for twill application {}. It is likely caused by " + "invalid CDAP program runtime extension.", programId, resourceReport == null ? "'unknown twill application'" : resourceReport.getApplicationId()); return null;
private int getTwillContainersUsed(TwillController controller) { if (controller.getResourceReport() == null) { return 0; } int count = 1; // 1 for app master container ResourceReport resourceReport = controller.getResourceReport(); for (Collection<TwillRunResources> resources : resourceReport.getResources().values()) { count += resources.size(); } return count; }
public static int numRunning(TwillController controller, String runnableName) { return controller.getResourceReport().getRunnableResources(runnableName).size(); }