@Override public boolean isPreemptable() { return getQueue().isPreemptable(); } }
/** * Updates the relevant tracking variables after a runnable app with the given * queue and user has been removed. */ public void untrackRunnableApp(FSAppAttempt app) { // Update usersRunnableApps String user = app.getUser(); int newUserNumRunning = usersNumRunnableApps.get(user) - 1; if (newUserNumRunning == 0) { usersNumRunnableApps.remove(user); } else { usersNumRunnableApps.put(user, newUserNumRunning); } // Update runnable app bookkeeping for queues FSLeafQueue queue = app.getQueue(); FSParentQueue parent = queue.getParent(); while (parent != null) { parent.decrementRunnableApps(); parent = parent.getParent(); } }
/** * Tracks the given new runnable app for purposes of maintaining max running * app limits. */ public void trackRunnableApp(FSAppAttempt app) { String user = app.getUser(); FSLeafQueue queue = app.getQueue(); // Increment running counts for all parent queues FSParentQueue parent = queue.getParent(); while (parent != null) { parent.incrementRunnableApps(); parent = parent.getParent(); } Integer userNumRunnable = usersNumRunnableApps.get(user); usersNumRunnableApps.put(user, (userNumRunnable == null ? 0 : userNumRunnable) + 1); }
/** * Tracks the given new runnable app for purposes of maintaining max running * app limits. */ public void trackRunnableApp(FSAppAttempt app) { String user = app.getUser(); FSLeafQueue queue = app.getQueue(); // Increment running counts for all parent queues FSParentQueue parent = queue.getParent(); while (parent != null) { parent.incrementRunnableApps(); parent = parent.getParent(); } Integer userNumRunnable = usersNumRunnableApps.get(user); usersNumRunnableApps.put(user, (userNumRunnable == null ? 0 : userNumRunnable) + 1); }
/** * Updates the relevant tracking variables after a runnable app with the given * queue and user has been removed. */ public void untrackRunnableApp(FSAppAttempt app) { // Update usersRunnableApps String user = app.getUser(); int newUserNumRunning = usersNumRunnableApps.get(user) - 1; if (newUserNumRunning == 0) { usersNumRunnableApps.remove(user); } else { usersNumRunnableApps.put(user, newUserNumRunning); } // Update runnable app bookkeeping for queues FSLeafQueue queue = app.getQueue(); FSParentQueue parent = queue.getParent(); while (parent != null) { parent.decrementRunnableApps(); parent = parent.getParent(); } }
/** * Updates the relevant tracking variables after a runnable app with the given * queue and user has been removed. */ public void untrackRunnableApp(FSAppAttempt app) { // Update usersRunnableApps String user = app.getUser(); int newUserNumRunning = usersNumRunnableApps.get(user) - 1; if (newUserNumRunning == 0) { usersNumRunnableApps.remove(user); } else { usersNumRunnableApps.put(user, newUserNumRunning); } // Update runnable app bookkeeping for queues FSLeafQueue queue = app.getQueue(); FSParentQueue parent = queue.getParent(); while (parent != null) { parent.decrementRunnableApps(); parent = parent.getParent(); } }
/** * Tracks the given new runnable app for purposes of maintaining max running * app limits. */ public void trackRunnableApp(FSAppAttempt app) { String user = app.getUser(); FSLeafQueue queue = app.getQueue(); // Increment running counts for all parent queues FSParentQueue parent = queue.getParent(); while (parent != null) { parent.incrementRunnableApps(); parent = parent.getParent(); } Integer userNumRunnable = usersNumRunnableApps.get(user); usersNumRunnableApps.put(user, (userNumRunnable == null ? 0 : userNumRunnable) + 1); }
/** * Helper method that checks if {@code usage} is strictly less than * {@code share}. */ private boolean isUsageBelowShare(Resource usage, Resource share) { return getQueue().getPolicy().getResourceCalculator().compare( scheduler.getClusterResource(), usage, share, true) < 0; }
private boolean isReservable(Resource capacity) { // Reserve only when the app is starved and the requested container size // is larger than the configured threshold return isStarved() && scheduler.isAtLeastReservationThreshold( getQueue().getPolicy().getResourceCalculator(), capacity); }
@Override public Resource assignContainer(FSSchedulerNode node) { if (isOverAMShareLimit()) { PendingAsk amAsk = appSchedulingInfo.getNextPendingAsk(); updateAMDiagnosticMsg(amAsk.getPerAllocationResource(), " exceeds maximum AM resource allowed)."); if (LOG.isDebugEnabled()) { LOG.debug("AM resource request: " + amAsk.getPerAllocationResource() + " exceeds maximum AM resource allowed, " + getQueue().dumpState()); } return Resources.none(); } return assignContainer(node, false); }
@Override public synchronized void recoverContainer(SchedulerNode node, RMContainer rmContainer) { try { writeLock.lock(); super.recoverContainer(node, rmContainer); if (!rmContainer.getState().equals(RMContainerState.COMPLETED)) { getQueue().incUsedResource(rmContainer.getContainer().getResource()); } // If not running unmanaged, the first container we recover is always // the AM. Set the amResource for this app and update the leaf queue's AM // usage if (!isAmRunning() && !getUnmanagedAM()) { Resource resource = rmContainer.getAllocatedResource(); setAMResource(resource); getQueue().addAMResourceUsage(resource); setAmRunning(true); } } finally { writeLock.unlock(); } }
/** * Whether the AM container for this app is over maxAMShare limit. */ private boolean isOverAMShareLimit() { // Check the AM resource usage for the leaf queue if (!isAmRunning() && !getUnmanagedAM()) { // Return true if we have not ask, or queue is not be able to run app's AM PendingAsk ask = appSchedulingInfo.getNextPendingAsk(); if (ask.getCount() == 0 || !getQueue().canRunAppAM( ask.getPerAllocationResource())) { return true; } } return false; }
/** * Helper method that computes the extent of fairshare starvation. * @return freshly computed fairshare starvation */ Resource fairShareStarvation() { long now = scheduler.getClock().getTime(); Resource threshold = Resources.multiply( getFairShare(), getQueue().getFairSharePreemptionThreshold()); Resource fairDemand = Resources.componentwiseMin(threshold, demand); // Check if the queue is starved for fairshare boolean starved = isUsageBelowShare(getResourceUsage(), fairDemand); if (!starved) { lastTimeAtFairShare = now; } if (!starved || now - lastTimeAtFairShare < getQueue().getFairSharePreemptionTimeout()) { fairshareStarvation = Resources.none(); } else { // The app has been starved for longer than preemption-timeout. fairshareStarvation = Resources.subtractFromNonNegative(fairDemand, getResourceUsage()); } return fairshareStarvation; }
final FSQueue fsQueue = getQueue(); SchedulingPolicy policy = fsQueue.getPolicy();
private void verifyAppRunnable(ApplicationAttemptId attId, boolean runnable) { FSAppAttempt app = scheduler.getSchedulerApp(attId); FSLeafQueue queue = app.getQueue(); assertEquals(runnable, queue.isRunnableApp(app)); assertEquals(!runnable, queue.isNonRunnableApp(app)); }
private void verifyAppRunnable(ApplicationAttemptId attId, boolean runnable) { FSAppAttempt app = scheduler.getSchedulerApp(attId); FSLeafQueue queue = app.getQueue(); assertEquals(runnable, queue.isRunnableApp(app)); assertEquals(!runnable, queue.isNonRunnableApp(app)); }
protected void warnOrKillContainer(RMContainer container) { ApplicationAttemptId appAttemptId = container.getApplicationAttemptId(); FSAppAttempt app = getSchedulerApp(appAttemptId); FSLeafQueue queue = app.getQueue(); LOG.info("Preempting container (prio=" + container.getContainer().getPriority() + "res=" + container.getContainer().getResource() + ") from queue " + queue.getName()); Long time = app.getContainerPreemptionTime(container); if (time != null) { // if we asked for preemption more than maxWaitTimeBeforeKill ms ago, // proceed with kill if (time + waitTimeBeforeKill < getClock().getTime()) { ContainerStatus status = SchedulerUtils.createPreemptedContainerStatus( container.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER); // TODO: Not sure if this ever actually adds this to the list of cleanup // containers on the RMNode (see SchedulerNode.releaseContainer()). completedContainer(container, status, RMContainerEventType.KILL); LOG.info("Killing container" + container + " (after waiting for premption for " + (getClock().getTime() - time) + "ms)"); } } else { // track the request in the FSAppAttempt itself app.addPreemption(container, getClock().getTime()); } }
protected void warnOrKillContainer(RMContainer container) { ApplicationAttemptId appAttemptId = container.getApplicationAttemptId(); FSAppAttempt app = getSchedulerApp(appAttemptId); FSLeafQueue queue = app.getQueue(); LOG.info("Preempting container (prio=" + container.getContainer().getPriority() + "res=" + container.getContainer().getResource() + ") from queue " + queue.getName()); Long time = app.getContainerPreemptionTime(container); if (time != null) { // if we asked for preemption more than maxWaitTimeBeforeKill ms ago, // proceed with kill if (time + waitTimeBeforeKill < getClock().getTime()) { ContainerStatus status = SchedulerUtils.createPreemptedContainerStatus( container.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER); // TODO: Not sure if this ever actually adds this to the list of cleanup // containers on the RMNode (see SchedulerNode.releaseContainer()). completedContainer(container, status, RMContainerEventType.KILL); LOG.info("Killing container" + container + " (after waiting for premption for " + (getClock().getTime() - time) + "ms)"); } } else { // track the request in the FSAppAttempt itself app.addPreemption(container, getClock().getTime()); } }