- Add the Codota plugin to your IDE and get smart completions
private void myMethod () {ArrayList a =
new ArrayList<String>()
- Smart code suggestions by Tabnine
}
syncKillableContainersFromScheduler(); cloneQueues(root, Resources .clone(nlm.getResourceByLabel(partitionToLookAt, clusterResources)), partitionToLookAt); this.leafQueueNames = ImmutableSet.copyOf(getLeafQueueNames( getQueueByPartition(CapacitySchedulerConfiguration.ROOT, RMNodeLabelsManager.NO_LABEL))); logToCSV(new ArrayList<>(leafQueueNames)); preemptOrkillSelectedContainerAfterWait(toPreemptPerSelector, currentTime); cleanupStaledPreemptionCandidates(currentTime);
@Override public void editSchedule() { CSQueue root = scheduler.getRootQueue(); Resource clusterResources = Resources.clone(scheduler.getClusterResource()); clusterResources = getNonLabeledResources(clusterResources); setNodeLabels(scheduler.getRMContext().getNodeLabelManager() .getNodeLabels()); containerBasedPreemptOrKill(root, clusterResources); }
public ProportionalCapacityPreemptionPolicy(Configuration config, RMContext context, CapacityScheduler scheduler, Clock clock) { init(config, context, scheduler); this.clock = clock; }
@Override public synchronized void editSchedule() { updateConfigIfNeeded(); long startTs = clock.getTime(); CSQueue root = scheduler.getRootQueue(); Resource clusterResources = Resources.clone(scheduler.getClusterResource()); containerBasedPreemptOrKill(root, clusterResources); if (LOG.isDebugEnabled()) { LOG.debug("Total time used=" + (clock.getTime() - startTs) + " ms."); } }
/** * This method recursively computes the ideal assignment of resources to each * level of the hierarchy. This ensures that leafs that are over-capacity but * with parents within capacity will not be preempted. Preemptions are allowed * within each subtree according to local over/under capacity. * * @param root the root of the cloned queue hierachy * @param totalPreemptionAllowed maximum amount of preemption allowed * @return a list of leaf queues updated with preemption targets */ private List<TempQueue> recursivelyComputeIdealAssignment( TempQueue root, Resource totalPreemptionAllowed) { List<TempQueue> leafs = new ArrayList<TempQueue>(); if (root.getChildren() != null && root.getChildren().size() > 0) { // compute ideal distribution at this level computeIdealResourceDistribution(rc, root.getChildren(), totalPreemptionAllowed, root.idealAssigned); // compute recursively for lower levels and build list of leafs for(TempQueue t : root.getChildren()) { leafs.addAll(recursivelyComputeIdealAssignment(t, totalPreemptionAllowed)); } } else { // we are in a leaf nothing to do, just return yourself return Collections.singletonList(root); } return leafs; }
@Test public void testIgnore() { ProportionalCapacityPreemptionPolicy policy = buildPolicy(Q_DATA_FOR_IGNORE); policy.editSchedule(); // don't correct imbalances without demand verify(mDisp, never()).handle(isA(ContainerPreemptEvent.class)); }
tRoot = cloneQueues(root, clusterResources); percentageClusterPreemptionAllowed); List<TempQueue> queues = recursivelyComputeIdealAssignment(tRoot, totalPreemptionAllowed); getContainersToPreempt(queues, clusterResources); logToCSV(queues);
isPreemptionToBalanceEnabled); buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig, true); policy.editSchedule(); IsPreemptionRequestFor(getAppAttemptId(2)))); assertEquals(60, policy.getQueuePartitions().get("a") .get("").getIdealAssigned().getMemorySize()); assertEquals(60, policy.getQueuePartitions().get("a") .get("").getIdealAssigned().getVirtualCores()); assertEquals(40, policy.getQueuePartitions().get("b") .get("").getIdealAssigned().getMemorySize()); assertEquals(40, policy.getQueuePartitions().get("b") .get("").getIdealAssigned().getVirtualCores());
@Before public void setup() { super.setup(); policy = new ProportionalCapacityPreemptionPolicy(rmContext, cs, mClock); }
new ArrayList<RMContainer>(app.getLiveContainers()); sortContainers(containers); if(isLabeledContainer(c)){ continue;
@Test public void testRefreshPreemptionProperties() throws Exception { ProportionalCapacityPreemptionPolicy policy = buildPolicy(Q_DATA_FOR_IGNORE); assertEquals( CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_MONITORING_INTERVAL, policy.getMonitoringInterval()); assertEquals( CapacitySchedulerConfiguration.DEFAULT_PREEMPTION_OBSERVE_ONLY, policy.isObserveOnly()); CapacitySchedulerConfiguration newConf = new CapacitySchedulerConfiguration(conf); long newMonitoringInterval = 5000; boolean newObserveOnly = true; newConf.setLong( CapacitySchedulerConfiguration.PREEMPTION_MONITORING_INTERVAL, newMonitoringInterval); newConf.setBoolean(CapacitySchedulerConfiguration.PREEMPTION_OBSERVE_ONLY, newObserveOnly); when(mCS.getConfiguration()).thenReturn(newConf); policy.editSchedule(); assertEquals(newMonitoringInterval, policy.getMonitoringInterval()); assertEquals(newObserveOnly, policy.isObserveOnly()); }
TempQueuePerPartition subq = cloneQueues(c, partitionResource, partitionToLookAt); addTempQueuePartition(ret); return ret;
Resource childrensPreemptable = Resource.newInstance(0, 0); for (CSQueue c : root.getChildQueues()) { TempQueue subq = cloneQueues(c, clusterResources); Resources.addTo(childrensPreemptable, subq.preemptableExtra); ret.addChild(subq);
@Test public void testContainerOrdering(){ List<RMContainer> containers = new ArrayList<RMContainer>(); ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance( ApplicationId.newInstance(TS, 10), 0); // create a set of containers RMContainer rm1 = mockContainer(appAttId, 5, mock(Resource.class), 3); RMContainer rm2 = mockContainer(appAttId, 3, mock(Resource.class), 3); RMContainer rm3 = mockContainer(appAttId, 2, mock(Resource.class), 2); RMContainer rm4 = mockContainer(appAttId, 1, mock(Resource.class), 2); RMContainer rm5 = mockContainer(appAttId, 4, mock(Resource.class), 1); // insert them in non-sorted order containers.add(rm3); containers.add(rm2); containers.add(rm1); containers.add(rm5); containers.add(rm4); // sort them ProportionalCapacityPreemptionPolicy.sortContainers(containers); // verify the "priority"-first, "reverse container-id"-second // ordering is enforced correctly assert containers.get(0).equals(rm1); assert containers.get(1).equals(rm2); assert containers.get(2).equals(rm3); assert containers.get(3).equals(rm4); assert containers.get(4).equals(rm5); }
computeFixpointAllocation(rc, tot_guarant, nonZeroGuarQueues, unassigned, false); computeFixpointAllocation(rc, tot_guarant, zeroGuarQueues, unassigned, true);
@Test public void testObserveOnly() { int[][] qData = new int[][]{ // / A B C { 100, 40, 40, 20 }, // abs { 100, 100, 100, 100 }, // maxCap { 100, 90, 10, 0 }, // used { 80, 10, 20, 50 }, // pending { 0, 0, 0, 0 }, // reserved { 2, 1, 1, 0 }, // apps { -1, 1, 1, 0 }, // req granularity { 3, 0, 0, 0 }, // subqueues }; conf.setBoolean(OBSERVE_ONLY, true); ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData); policy.editSchedule(); // verify even severe imbalance not affected verify(mDisp, never()).handle(isA(ContainerPreemptEvent.class)); }
tRoot = cloneQueues(root, clusterResources); percentageClusterPreemptionAllowed); List<TempQueue> queues = recursivelyComputeIdealAssignment(tRoot, totalPreemptionAllowed); getContainersToPreempt(queues, clusterResources); logToCSV(queues);
policy.editSchedule(); getAppAttemptId(3)))); assertEquals(16, policy.getQueuePartitions().get("a") .get("").getIdealAssigned().getMemorySize()); assertEquals(42, policy.getQueuePartitions().get("b") .get("").getIdealAssigned().getMemorySize()); assertEquals(42, policy.getQueuePartitions().get("c") .get("").getIdealAssigned().getMemorySize());
@Before public void setup() { super.setup(); rc = new DominantResourceCalculator(); when(cs.getResourceCalculator()).thenReturn(rc); policy = new ProportionalCapacityPreemptionPolicy(rmContext, cs, mClock); }
new ArrayList<RMContainer>(app.getLiveContainers()); sortContainers(containers); if(isLabeledContainer(c)){ continue;