/** * Checks to see if the report has loaded. * @param report - The {@link ResourceReport} for this Twill Application. * @return Return true if the report is null or incomplete. Return false if the report is completely loaded. */ private boolean reportIsLoading(@Nullable final ResourceReport report) { if(report == null) { return true; } final String yarnApplicationID = report.getApplicationId(); final Collection<TwillRunResources> runnableResources = report.getResources().get(PeriodicNotificationTwillRunnable.TWILL_RUNNABLE_NAME); if(runnableResources == null || runnableResources.isEmpty()) { LOG.info("Received Resource Report for YARN ApplicationID: {}, runnable resources are still loading...", yarnApplicationID); return true; } else { LOG.info("Received Resource Report for YARN ApplicationID: {}, runnable resources are loaded.", yarnApplicationID); return false; } }
/** * Terminates all instances of the {@link PeriodicNotificationTwillApp} on the YARN cluster. */ public void stopApp() { LOG.info("Stopping any running instances..."); int counter = 0; // It is possible that we have launched multiple instances of the app. For now, stop them all, one at a time. for(final TwillController c : twillRunner.lookup(PeriodicNotificationTwillApp.APPLICATION_NAME)) { final ResourceReport report = c.getResourceReport(); LOG.info("Attempting to stop {} with YARN ApplicationId: {} and Twill RunId: {}", PeriodicNotificationTwillApp.APPLICATION_NAME, report.getApplicationId(), c.getRunId()); Futures.getUnchecked(c.terminate()); LOG.info("Stopped {} with YARN ApplicationId: {} and Twill RunId: {}", PeriodicNotificationTwillApp.APPLICATION_NAME, report.getApplicationId(), c.getRunId()); counter++; } LOG.info("Stopped {} instance(s) of {}", counter, PeriodicNotificationTwillApp.APPLICATION_NAME); }
@Override public JsonElement serialize(ResourceReport src, Type typeOfSrc, JsonSerializationContext context) { JsonObject json = new JsonObject(); json.addProperty("appMasterId", src.getApplicationId()); json.add("appMasterResources", context.serialize(src.getAppMasterResources(), TwillRunResources.class)); json.add("runnableResources", context.serialize( src.getResources(), new TypeToken<Map<String, Collection<TwillRunResources>>>() { }.getType())); json.add("services", context.serialize( src.getServices(), new TypeToken<List<String>>() { }.getType())); return json; }
@Override public JsonElement serialize(ResourceReport src, Type typeOfSrc, JsonSerializationContext context) { JsonObject json = new JsonObject(); json.addProperty("appMasterId", src.getApplicationId()); json.add("appMasterResources", context.serialize(src.getAppMasterResources(), TwillRunResources.class)); json.add("runnableResources", context.serialize( src.getResources(), new TypeToken<Map<String, Collection<TwillRunResources>>>() { }.getType())); json.add("services", context.serialize( src.getServices(), new TypeToken<List<String>>() { }.getType())); return json; }
LOG.error("Unable to create ProgramController for program {} for twill application {}. It is likely caused by " + "invalid CDAP program runtime extension.", programId, resourceReport == null ? "'unknown twill application'" : resourceReport.getApplicationId()); return null;
LOG.error("Unable to create ProgramController for program {} for twill application {}. It is likely caused by " + "invalid CDAP program runtime extension.", programId, resourceReport == null ? "'unknown twill application'" : resourceReport.getApplicationId()); return null;
DistributedProgramLiveInfo liveInfo = new DistributedProgramLiveInfo(program, report.getApplicationId());
String appId = getResourceReport(controller, -1).getApplicationId(); Preconditions.checkNotNull(appId, "Failed to retrieve YARN app ID from Twill"); CuratorUtil.putData(getAppCurator(config), ZookeeperPath.YARN_APP_ID,
String appId = getResourceReport(controller, -1).getApplicationId(); Objects.requireNonNull(appId, "Failed to retrieve YARN app ID from Twill"); CuratorUtil.putData(getAppCurator(config), ZookeeperPath.YARN_APP_ID,
DistributedProgramLiveInfo liveInfo = new DistributedProgramLiveInfo(program, report.getApplicationId());
String appId = getResourceReport(controller, -1).getApplicationId(); Objects.requireNonNull(appId, "Failed to retrieve YARN app ID from Twill"); CuratorUtil.putData(getAppCurator(config), ZookeeperPath.YARN_APP_ID,
private void waitForContainers(TwillController controller, int count, long timeout, TimeUnit timeoutUnit) throws Exception { Stopwatch stopwatch = new Stopwatch(); stopwatch.start(); int yarnContainers = 0; int twillContainers = 0; do { if (controller.getResourceReport() != null) { yarnContainers = getApplicationResourceReport(controller.getResourceReport().getApplicationId()).getNumUsedContainers(); twillContainers = getTwillContainersUsed(controller); if (yarnContainers == count && twillContainers == count) { return; } } TimeUnit.SECONDS.sleep(1); } while (stopwatch.elapsedTime(timeoutUnit) < timeout); throw new TimeoutException("Timeout reached while waiting for num containers to be " + count + ". Yarn containers = " + yarnContainers + ", Twill containers = " + twillContainers); }