@Nullable @Override public ResourceReport call() throws Exception { return delegate.getResourceReport(); } });
@Nullable @Override public ResourceReport call() throws Exception { return delegate.getResourceReport(); } });
@Override public int getInstances() { Iterable<TwillController> twillControllerList = twillRunnerService.lookup(Constants.Service.MASTER_SERVICES); int instances = 0; if (twillControllerList != null) { for (TwillController twillController : twillControllerList) { if (twillController.getResourceReport() != null) { instances = twillController.getResourceReport().getRunnableResources(serviceName).size(); } } } return instances; }
@Override public int getInstances() { Iterable<TwillController> twillControllerList = twillRunnerService.lookup(Constants.Service.MASTER_SERVICES); int instances = 0; if (twillControllerList != null) { for (TwillController twillController : twillControllerList) { if (twillController.getResourceReport() != null) { instances = twillController.getResourceReport().getRunnableResources(serviceName).size(); } } } return instances; }
private int getNumberOfProvisionedInstances(String flowletId) { return twillController.getResourceReport().getRunnableResources(flowletId).size(); } }
private int getNumberOfProvisionedInstances(String flowletId) { return twillController.getResourceReport().getRunnableResources(flowletId).size(); }
public static int numRunning(TwillController controller, String runnableName) { return controller.getResourceReport().getRunnableResources(runnableName).size(); }
public static int numRunning(TwillController controller, String runnableName) { return controller.getResourceReport().getRunnableResources(runnableName).size(); }
public static int numRunning(TwillController controller, String runnableName) { return controller.getResourceReport().getRunnableResources(runnableName).size(); }
@Override public Map<String, Integer> getFlowInfo(String flowName) { Map<String, Integer> flowletInfo = Maps.newHashMap(); try { Iterable<TwillController> controllers = lookupFlow(flowName); for (TwillController controller : controllers) { ResourceReport report = controller.getResourceReport(); sleepForZK(report); for (Map.Entry<String, Collection<TwillRunResources>> entry : report.getResources().entrySet()) { flowletInfo.put(entry.getKey(), entry.getValue().size()); } } } catch (Exception e) { LOG.warn(e.getMessage(), e); } return flowletInfo; }
@Override public List<String> getServices(String flowName) { List<String> services = Lists.newArrayList(); try { Iterable<TwillController> controllers = lookupFlow(flowName); for (TwillController controller : controllers) { ResourceReport report = controller.getResourceReport(); sleepForZK(report); services.addAll(report.getServices()); } } catch (Exception e) { LOG.warn(e.getMessage(), e); } return services; }
private int getTwillContainersUsed(TwillController controller) { if (controller.getResourceReport() == null) { return 0; } int count = 1; // 1 for app master container ResourceReport resourceReport = controller.getResourceReport(); for (Collection<TwillRunResources> resources : resourceReport.getResources().values()) { count += resources.size(); } return count; }
/** * Terminates all instances of the {@link PeriodicNotificationTwillApp} on the YARN cluster. */ public void stopApp() { LOG.info("Stopping any running instances..."); int counter = 0; // It is possible that we have launched multiple instances of the app. For now, stop them all, one at a time. for(final TwillController c : twillRunner.lookup(PeriodicNotificationTwillApp.APPLICATION_NAME)) { final ResourceReport report = c.getResourceReport(); LOG.info("Attempting to stop {} with YARN ApplicationId: {} and Twill RunId: {}", PeriodicNotificationTwillApp.APPLICATION_NAME, report.getApplicationId(), c.getRunId()); Futures.getUnchecked(c.terminate()); LOG.info("Stopped {} with YARN ApplicationId: {} and Twill RunId: {}", PeriodicNotificationTwillApp.APPLICATION_NAME, report.getApplicationId(), c.getRunId()); counter++; } LOG.info("Stopped {} instance(s) of {}", counter, PeriodicNotificationTwillApp.APPLICATION_NAME); }
private ResourceReport getResourceReport(TwillController controller, long timeoutMillis) { ResourceReport report = controller.getResourceReport(); Stopwatch stopwatch = new Stopwatch(); while (report == null && stopwatch.elapsedMillis() < timeoutMillis) { Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS); report = controller.getResourceReport(); } Assert.assertNotNull(report); return report; } }
@Override public void reportResources() { for (TwillRunner.LiveInfo info : twillRunner.lookupLive()) { Map<String, String> metricContext = getMetricContext(info); if (metricContext == null) { continue; } // will have multiple controllers if there are multiple runs of the same application for (TwillController controller : info.getControllers()) { ResourceReport report = controller.getResourceReport(); if (report == null) { continue; } int memory = report.getAppMasterResources().getMemoryMB(); int vcores = report.getAppMasterResources().getVirtualCores(); Map<String, String> runContext = ImmutableMap.<String, String>builder() .putAll(metricContext) .put(Constants.Metrics.Tag.RUN_ID, controller.getRunId().getId()).build(); sendMetrics(runContext, 1, memory, vcores); } } reportYarnResources(); }
@Override public void reportResources() { for (TwillRunner.LiveInfo info : twillRunner.lookupLive()) { Map<String, String> metricContext = getMetricContext(info); if (metricContext == null) { continue; } // will have multiple controllers if there are multiple runs of the same application for (TwillController controller : info.getControllers()) { ResourceReport report = controller.getResourceReport(); if (report == null) { continue; } int memory = report.getAppMasterResources().getMemoryMB(); int vcores = report.getAppMasterResources().getVirtualCores(); Map<String, String> runContext = ImmutableMap.<String, String>builder() .putAll(metricContext) .put(Constants.Metrics.Tag.RUN_ID, controller.getRunId().getId()).build(); sendMetrics(runContext, 1, memory, vcores); } } reportYarnResources(); }
private void waitForContainers(TwillController controller, int count, long timeout, TimeUnit timeoutUnit) throws Exception { Stopwatch stopwatch = new Stopwatch(); stopwatch.start(); int yarnContainers = 0; int twillContainers = 0; do { if (controller.getResourceReport() != null) { yarnContainers = getApplicationResourceReport(controller.getResourceReport().getApplicationId()).getNumUsedContainers(); twillContainers = getTwillContainersUsed(controller); if (yarnContainers == count && twillContainers == count) { return; } } TimeUnit.SECONDS.sleep(1); } while (stopwatch.elapsedTime(timeoutUnit) < timeout); throw new TimeoutException("Timeout reached while waiting for num containers to be " + count + ". Yarn containers = " + yarnContainers + ", Twill containers = " + twillContainers); }
private boolean waitForDebugPort(TwillController controller, String runnable, int timeLimit) throws InterruptedException { long millis = 0; while (millis < 1000 * timeLimit) { ResourceReport report = controller.getResourceReport(); if (report == null || report.getRunnableResources(runnable) == null) { continue; } for (TwillRunResources resources : report.getRunnableResources(runnable)) { if (resources.getDebugPort() != null) { return true; } } TimeUnit.MILLISECONDS.sleep(100); millis += 100; } return false; }
private void waitForInstance(TwillController controller, String runnable, String yarnInstanceId, long timeout, TimeUnit timeoutUnit) throws InterruptedException, TimeoutException { Stopwatch stopwatch = new Stopwatch(); stopwatch.start(); do { ResourceReport report = controller.getResourceReport(); if (report != null && report.getRunnableResources(runnable) != null) { for (TwillRunResources resources : report.getRunnableResources(runnable)) { if (resources.getContainerId().endsWith(yarnInstanceId)) { return; } } } TimeUnit.SECONDS.sleep(1); } while (stopwatch.elapsedTime(timeoutUnit) < timeout); throw new TimeoutException("Timeout reached while waiting for runnable " + runnable + " instance " + yarnInstanceId); }
private boolean waitForLogLevel(TwillController controller, String runnable, long timeout, TimeUnit timeoutUnit, @Nullable LogEntry.Level expected) throws InterruptedException { Stopwatch stopwatch = new Stopwatch(); stopwatch.start(); do { ResourceReport report = controller.getResourceReport(); if (report == null || report.getRunnableResources(runnable) == null) { continue; } for (TwillRunResources resources : report.getRunnableResources(runnable)) { LogEntry.Level level = resources.getLogLevels().get(Logger.ROOT_LOGGER_NAME); if (expected == level) { return true; } } TimeUnit.MILLISECONDS.sleep(100); } while (stopwatch.elapsedTime(timeoutUnit) < timeout); return false; } }