/** * Whether this app's starvation should be considered. */ boolean shouldCheckForStarvation() { return scheduler.getClock().getTime() >= nextStarvationCheck; }
/** * Notify this app that preemption has been triggered to make room for * outstanding demand. The app should not be considered starved until after * the specified delay. * * @param delayBeforeNextStarvationCheck duration to wait */ void preemptionTriggered(long delayBeforeNextStarvationCheck) { nextStarvationCheck = scheduler.getClock().getTime() + delayBeforeNextStarvationCheck; }
public FSAppAttempt(FairScheduler scheduler, ApplicationAttemptId applicationAttemptId, String user, FSLeafQueue queue, ActiveUsersManager activeUsersManager, RMContext rmContext) { super(applicationAttemptId, user, queue, activeUsersManager, rmContext); this.scheduler = scheduler; this.startTime = scheduler.getClock().getTime(); this.lastTimeAtFairShare = this.startTime; this.appPriority = Priority.newInstance(1); }
@Override public void run() { while (!Thread.currentThread().isInterrupted()) { try { Thread.sleep(updateInterval); long start = getClock().getTime(); update(); preemptTasksIfNecessary(); long duration = getClock().getTime() - start; fsOpDurations.addUpdateThreadRunDuration(duration); } catch (InterruptedException ie) { LOG.warn("Update thread interrupted. Exiting."); return; } catch (Exception e) { LOG.error("Exception in fair scheduler UpdateThread", e); } } } }
public FSLeafQueue(String name, FairScheduler scheduler, FSParentQueue parent) { super(name, scheduler, parent); this.lastTimeAtMinShare = scheduler.getClock().getTime(); this.lastTimeAtFairShareThreshold = scheduler.getClock().getTime(); activeUsersManager = new ActiveUsersManager(getMetrics()); amResourceUsage = Resource.newInstance(0, 0); }
@Override public void run() { while (!Thread.currentThread().isInterrupted()) { try { Thread.sleep(updateInterval); long start = getClock().getTime(); update(); preemptTasksIfNecessary(); long duration = getClock().getTime() - start; fsOpDurations.addUpdateThreadRunDuration(duration); } catch (InterruptedException ie) { LOG.warn("Update thread interrupted. Exiting."); return; } catch (Exception e) { LOG.error("Exception in fair scheduler UpdateThread", e); } } } }
public FSLeafQueue(String name, FairScheduler scheduler, FSParentQueue parent) { super(name, scheduler, parent); this.lastTimeAtMinShare = scheduler.getClock().getTime(); this.lastTimeAtFairShareThreshold = scheduler.getClock().getTime(); activeUsersManager = new ActiveUsersManager(getMetrics()); amResourceUsage = Resource.newInstance(0, 0); }
@Override protected void nodeUpdate(RMNode nm) { writeLock.lock(); try { long start = getClock().getTime(); super.nodeUpdate(nm); FSSchedulerNode fsNode = getFSSchedulerNode(nm.getNodeID()); attemptScheduling(fsNode); long duration = getClock().getTime() - start; fsOpDurations.addNodeUpdateDuration(duration); } finally { writeLock.unlock(); } }
public FSAppAttempt(FairScheduler scheduler, ApplicationAttemptId applicationAttemptId, String user, FSLeafQueue queue, ActiveUsersManager activeUsersManager, RMContext rmContext) { super(applicationAttemptId, user, queue, activeUsersManager, rmContext); this.scheduler = scheduler; this.startTime = scheduler.getClock().getTime(); this.priority = Priority.newInstance(1); this.resourceWeights = new ResourceWeights(); }
public FSAppAttempt(FairScheduler scheduler, ApplicationAttemptId applicationAttemptId, String user, FSLeafQueue queue, ActiveUsersManager activeUsersManager, RMContext rmContext) { super(applicationAttemptId, user, queue, activeUsersManager, rmContext); this.scheduler = scheduler; this.startTime = scheduler.getClock().getTime(); this.priority = Priority.newInstance(1); this.resourceWeights = new ResourceWeights(); }
/** * Update the preemption fields for the queue, i.e. the times since last was * at its guaranteed share and over its fair share threshold. */ public void updateStarvationStats() { long now = scheduler.getClock().getTime(); if (!isStarvedForMinShare()) { setLastTimeAtMinShare(now); } if (!isStarvedForFairShare()) { setLastTimeAtFairShareThreshold(now); } }
/** * Update the preemption fields for the queue, i.e. the times since last was * at its guaranteed share and over its fair share threshold. */ public void updateStarvationStats() { long now = scheduler.getClock().getTime(); if (!isStarvedForMinShare()) { setLastTimeAtMinShare(now); } if (!isStarvedForFairShare()) { setLastTimeAtFairShareThreshold(now); } }
public FSLeafQueue(String name, FairScheduler scheduler, FSParentQueue parent) { super(name, scheduler, parent); this.context = scheduler.getContext(); this.lastTimeAtMinShare = scheduler.getClock().getTime(); activeUsersManager = new ActiveUsersManager(getMetrics()); amResourceUsage = Resource.newInstance(0, 0); getMetrics().setAMResourceUsage(amResourceUsage); }
@Before public void setup() throws Exception { Configuration conf = new Configuration(); clock = new TestFairScheduler.MockClock(); scheduler = mock(FairScheduler.class); when(scheduler.getConf()).thenReturn( new FairSchedulerConfiguration(conf)); when(scheduler.getClock()).thenReturn(clock); AllocationConfiguration allocConf = new AllocationConfiguration( conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); queueManager = new QueueManager(scheduler); queueManager.initialize(conf); queueMaxApps = allocConf.queueMaxApps; userMaxApps = allocConf.userMaxApps; maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler); appNum = 0; rmContext = mock(RMContext.class); when(rmContext.getEpoch()).thenReturn(0L); }
@Before public void setUp() throws Exception { conf = new FairSchedulerConfiguration(); FairScheduler scheduler = mock(FairScheduler.class); AllocationConfiguration allocConf = new AllocationConfiguration(conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); when(scheduler.getResourceCalculator()).thenReturn( new DefaultResourceCalculator()); SystemClock clock = SystemClock.getInstance(); when(scheduler.getClock()).thenReturn(clock); notEmptyQueues = new HashSet<FSQueue>(); queueManager = new QueueManager(scheduler) { @Override public boolean isEmpty(FSQueue queue) { return !notEmptyQueues.contains(queue); } }; FSQueueMetrics.forQueue("root", null, true, conf); queueManager.initialize(conf); }
@Before public void setUp() throws Exception { conf = new FairSchedulerConfiguration(); FairScheduler scheduler = mock(FairScheduler.class); AllocationConfiguration allocConf = new AllocationConfiguration(conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); SystemClock clock = new SystemClock(); when(scheduler.getClock()).thenReturn(clock); notEmptyQueues = new HashSet<FSQueue>(); queueManager = new QueueManager(scheduler) { @Override public boolean isEmpty(FSQueue queue) { return !notEmptyQueues.contains(queue); } }; FSQueueMetrics.forQueue("root", null, true, conf); queueManager.initialize(conf); }
@Test public void testEmptyChildQueues() throws Exception { FairSchedulerConfiguration conf = new FairSchedulerConfiguration(); FairScheduler scheduler = mock(FairScheduler.class); AllocationConfiguration allocConf = new AllocationConfiguration(conf); when(scheduler.getAllocationConfiguration()).thenReturn(allocConf); when(scheduler.getConf()).thenReturn(conf); when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(1, 1)); when(scheduler.getResourceCalculator()).thenReturn( new DefaultResourceCalculator()); SystemClock clock = SystemClock.getInstance(); when(scheduler.getClock()).thenReturn(clock); QueueManager queueManager = new QueueManager(scheduler); queueManager.initialize(conf); FSQueue testQueue = queueManager.getLeafQueue("test", true); FairSchedulerQueueInfo queueInfo = new FairSchedulerQueueInfo(testQueue, scheduler); Collection<FairSchedulerQueueInfo> childQueues = queueInfo.getChildQueues(); Assert.assertNotNull(childQueues); Assert.assertEquals("Child QueueInfo was not empty", 0, childQueues.size()); } }
/** * Helper method to compute the amount of minshare starvation. * * @return the extent of minshare starvation */ private Resource minShareStarvation() { // If demand < minshare, we should use demand to determine starvation Resource starvation = Resources.componentwiseMin(getMinShare(), getDemand()); Resources.subtractFromNonNegative(starvation, getResourceUsage()); boolean starved = !Resources.isNone(starvation); long now = scheduler.getClock().getTime(); if (!starved) { // Record that the queue is not starved setLastTimeAtMinShare(now); } if (now - lastTimeAtMinShare < getMinSharePreemptionTimeout()) { // the queue is not starved for the preemption timeout starvation = Resources.clone(Resources.none()); } return starvation; }
@Before public void setup() { mockClock = new MockClock(); conf = createConfiguration(); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); scheduler.setClock(mockClock); assertTrue(scheduler.isContinuousSchedulingEnabled()); assertEquals( FairSchedulerConfiguration.DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS, scheduler.getContinuousSchedulingSleepMs()); assertEquals(mockClock, scheduler.getClock()); }
@SuppressWarnings("deprecation") @Before public void setup() { mockClock = new ControlledClock(); conf = createConfiguration(); resourceManager = new MockRM(conf); resourceManager.start(); scheduler = (FairScheduler) resourceManager.getResourceScheduler(); scheduler.setClock(mockClock); assertTrue(scheduler.isContinuousSchedulingEnabled()); assertEquals( FairSchedulerConfiguration.DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS, scheduler.getContinuousSchedulingSleepMs()); assertEquals(mockClock, scheduler.getClock()); }