/** * Checks to see if the report has loaded. * @param report - The {@link ResourceReport} for this Twill Application. * @return Return true if the report is null or incomplete. Return false if the report is completely loaded. */ private boolean reportIsLoading(@Nullable final ResourceReport report) { if(report == null) { return true; } final String yarnApplicationID = report.getApplicationId(); final Collection<TwillRunResources> runnableResources = report.getResources().get(PeriodicNotificationTwillRunnable.TWILL_RUNNABLE_NAME); if(runnableResources == null || runnableResources.isEmpty()) { LOG.info("Received Resource Report for YARN ApplicationID: {}, runnable resources are still loading...", yarnApplicationID); return true; } else { LOG.info("Received Resource Report for YARN ApplicationID: {}, runnable resources are loaded.", yarnApplicationID); return false; } }
@Override public Map<String, Integer> getFlowInfo(String flowName) { Map<String, Integer> flowletInfo = Maps.newHashMap(); try { Iterable<TwillController> controllers = lookupFlow(flowName); for (TwillController controller : controllers) { ResourceReport report = controller.getResourceReport(); sleepForZK(report); for (Map.Entry<String, Collection<TwillRunResources>> entry : report.getResources().entrySet()) { flowletInfo.put(entry.getKey(), entry.getValue().size()); } } } catch (Exception e) { LOG.warn(e.getMessage(), e); } return flowletInfo; }
@Override public JsonElement serialize(ResourceReport src, Type typeOfSrc, JsonSerializationContext context) { JsonObject json = new JsonObject(); json.addProperty("appMasterId", src.getApplicationId()); json.add("appMasterResources", context.serialize(src.getAppMasterResources(), TwillRunResources.class)); json.add("runnableResources", context.serialize( src.getResources(), new TypeToken<Map<String, Collection<TwillRunResources>>>() { }.getType())); json.add("services", context.serialize( src.getServices(), new TypeToken<List<String>>() { }.getType())); return json; }
@Override public JsonElement serialize(ResourceReport src, Type typeOfSrc, JsonSerializationContext context) { JsonObject json = new JsonObject(); json.addProperty("appMasterId", src.getApplicationId()); json.add("appMasterResources", context.serialize(src.getAppMasterResources(), TwillRunResources.class)); json.add("runnableResources", context.serialize( src.getResources(), new TypeToken<Map<String, Collection<TwillRunResources>>>() { }.getType())); json.add("services", context.serialize( src.getServices(), new TypeToken<List<String>>() { }.getType())); return json; }
@Override public void setInstances(String flowName, String flowletName, int instanceCount) { try { Iterable<TwillController> controllers = lookupFlow(flowName); for (TwillController controller : controllers) { ResourceReport report = controller.getResourceReport(); sleepForZK(report); int oldInstances = report.getResources().get(flowletName).size(); Program program = Programs.create(location.append(flowName)); Multimap<String, QueueName> consumerQueues = FlowUtils.configureQueue(program, program.getSpecification(), queueAdmin); DistributedFlowletInstanceUpdater instanceUpdater = new DistributedFlowletInstanceUpdater( program, controller, queueAdmin, consumerQueues); FlowTwillProgramController flowController = new FlowTwillProgramController(program.getName(), controller, instanceUpdater); Map<String, String> instanceOptions = Maps.newHashMap(); instanceOptions.put("flowlet", flowletName); instanceOptions.put("newInstances", String.valueOf(instanceCount)); instanceOptions.put("oldInstances", String.valueOf(oldInstances)); flowController.command(ProgramOptionConstants.FLOWLET_INSTANCES, instanceOptions).get(); } } catch (Exception e) { LOG.warn(e.getMessage(), e); } }
private int getTwillContainersUsed(TwillController controller) { if (controller.getResourceReport() == null) { return 0; } int count = 1; // 1 for app master container ResourceReport resourceReport = controller.getResourceReport(); for (Collection<TwillRunResources> resources : resourceReport.getResources().values()) { count += resources.size(); } return count; }
for (Map.Entry<String, Collection<TwillRunResources>> entry : report.getResources().entrySet()) { for (TwillRunResources resources : entry.getValue()) { liveInfo.addContainer(new ContainerInfo(containerType,
Containers.ContainerType.valueOf(program.getType().name()); for (Map.Entry<String, Collection<TwillRunResources>> entry : report.getResources().entrySet()) { for (TwillRunResources resources : entry.getValue()) { liveInfo.addContainer(new ContainerInfo(containerType,
@Override public SystemServiceLiveInfo getLiveInfo() { SystemServiceLiveInfo.Builder builder = SystemServiceLiveInfo.builder(); Iterable<TwillController> twillControllerList = twillRunnerService.lookup(Constants.Service.MASTER_SERVICES); if (twillControllerList == null) { return builder.build(); } for (TwillController twillController : twillControllerList) { if (twillController.getResourceReport() == null) { continue; } ResourceReport resourceReport = twillController.getResourceReport(); Collection<TwillRunResources> runResources = resourceReport.getResources().get(serviceName); for (TwillRunResources resources : runResources) { Containers.ContainerInfo containerInfo = new Containers.ContainerInfo( Containers.ContainerType.SYSTEM_SERVICE, serviceName, resources.getInstanceId(), resources.getContainerId(), resources.getHost(), resources.getMemoryMB(), resources.getVirtualCores(), resources.getDebugPort()); builder.addContainer(containerInfo); } } return builder.build(); }
@Override public SystemServiceLiveInfo getLiveInfo() { SystemServiceLiveInfo.Builder builder = SystemServiceLiveInfo.builder(); Iterable<TwillController> twillControllerList = twillRunnerService.lookup(Constants.Service.MASTER_SERVICES); if (twillControllerList == null) { return builder.build(); } for (TwillController twillController : twillControllerList) { if (twillController.getResourceReport() == null) { continue; } ResourceReport resourceReport = twillController.getResourceReport(); Collection<TwillRunResources> runResources = resourceReport.getResources().get(serviceName); for (TwillRunResources resources : runResources) { Containers.ContainerInfo containerInfo = new Containers.ContainerInfo( Containers.ContainerType.SYSTEM_SERVICE, serviceName, resources.getInstanceId(), resources.getContainerId(), resources.getHost(), resources.getMemoryMB(), resources.getVirtualCores(), resources.getDebugPort()); builder.addContainer(containerInfo); } } return builder.build(); }
ResourceReport report = getResourceReport(controller, 10000); Map<String, Collection<TwillRunResources>> usedResources = report.getResources(); Assert.assertEquals(2, usedResources.keySet().size()); Assert.assertTrue(usedResources.containsKey("echo1")); usedResources = report.getResources(); Assert.assertEquals(2, usedResources.keySet().size()); Assert.assertTrue(usedResources.containsKey("echo1")); Assert.assertEquals(report.getResources(), newReport.getResources());