org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.editSchedule java code examples

@Test
public void testIgnore() {
 ProportionalCapacityPreemptionPolicy policy =
   buildPolicy(Q_DATA_FOR_IGNORE);
 policy.editSchedule();
 // don't correct imbalances without demand
 verify(mDisp, never()).handle(isA(ContainerPreemptEvent.class));
}

@Test
public void testObserveOnly() {
 int[][] qData = new int[][]{
  //  /   A   B   C
  { 100, 40, 40, 20 },  // abs
  { 100, 100, 100, 100 },  // maxCap
  { 100, 90, 10,  0 },  // used
  {  80, 10, 20, 50 },  // pending
  {   0,  0,  0,  0 },  // reserved
  {   2,  1,  1,  0 },  // apps
  {  -1,  1,  1,  0 },  // req granularity
  {   3,  0,  0,  0 },  // subqueues
 };
 conf.setBoolean(OBSERVE_ONLY, true);
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // verify even severe imbalance not affected
 verify(mDisp, never()).handle(isA(ContainerPreemptEvent.class));
}

@Test
public void testIgnore() {
 int[][] qData = new int[][]{
  //  /   A   B   C
  { 100, 40, 40, 20 },  // abs
  { 100, 100, 100, 100 },  // maxCap
  { 100,  0, 60, 40 },  // used
  {   0,  0,  0,  0 },  // pending
  {   0,  0,  0,  0 },  // reserved
  {   3,  1,  1,  1 },  // apps
  {  -1,  1,  1,  1 },  // req granularity
  {   3,  0,  0,  0 },  // subqueues
 };
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // don't correct imbalances without demand
 verify(mDisp, never()).handle(isA(ContainerPreemptEvent.class));
}

@Test
public void testProportionalPreemption() {
 int[][] qData = new int[][]{
  //  /   A   B   C  D
  { 100, 10, 40, 20, 30 },  // abs
  { 100, 100, 100, 100, 100 },  // maxCap
  { 100, 30, 60, 10,  0 },  // used
  {  45, 20,  5, 20,  0 },  // pending
  {   0,  0,  0,  0,  0 },  // reserved
  {   3,  1,  1,  1,  0 },  // apps
  {  -1,  1,  1,  1,  1 },  // req granularity
  {   4,  0,  0,  0,  0 },  // subqueues
 };
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 verify(mDisp, times(16)).handle(argThat(new IsPreemptionRequestFor(appA)));
}

@Test
public void testHierarchicalWithReserved() {
 int[][] qData = new int[][] {
   //  /    A   B   C    D   E   F
   { 200, 100, 50, 50, 100, 10, 90 },  // abs
   { 200, 200, 200, 200, 200, 200, 200 },  // maxCap
   { 200, 110, 60, 50,  90, 90,  0 },  // used
   {  10,   0,  0,  0,  10,  0, 10 },  // pending
   {  40,  25,  15, 10, 15,  15,  0 },  // reserved
   {   4,   2,  1,  1,   2,  1,  1 },  // apps
   {  -1,  -1,  1,  1,  -1,  1,  1 },  // req granularity
   {   2,   2,  0,  0,   2,  0,  0 },  // subqueues
 };
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // verify capacity taken from A1, not B1 despite B1 being far over
 // its absolute guaranteed capacity
 verify(mDisp, times(10)).handle(argThat(new IsPreemptionRequestFor(appA)));
}

@Test
public void testMaxCap() {
 int[][] qData = new int[][]{
   //  /   A   B   C
   { 100, 40, 40, 20 },  // abs
   { 100, 100, 45, 100 },  // maxCap
   { 100, 55, 45,  0 },  // used
   {  20, 10, 10,  0 },  // pending
   {   0,  0,  0,  0 },  // reserved
   {   2,  1,  1,  0 },  // apps
   {  -1,  1,  1,  0 },  // req granularity
   {   3,  0,  0,  0 },  // subqueues
  };
  ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
  policy.editSchedule();
  // despite the imbalance, since B is at maxCap, do not correct
  verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appA)));
}

@Test
public void testPreemptCycle() {
 int[][] qData = new int[][]{
  //  /   A   B   C
  { 100, 40, 40, 20 },  // abs
  { 100, 100, 100, 100 },  // maxCap
  { 100,  0, 60, 40 },  // used
  {  10, 10,  0,  0 },  // pending
  {   0,  0,  0,  0 },  // reserved
  {   3,  1,  1,  1 },  // apps
  {  -1,  1,  1,  1 },  // req granularity
  {   3,  0,  0,  0 },  // subqueues
 };
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // ensure all pending rsrc from A get preempted from other queues
 verify(mDisp, times(10)).handle(argThat(new IsPreemptionRequestFor(appC)));
}

@Test
public void testOverCapacityImbalance() {
 int[][] qData = new int[][]{
  //  /   A   B   C
  { 100, 40, 40, 20 },  // abs
  { 100, 100, 100, 100 },  // maxCap
  { 100, 55, 45,  0 },  // used
  {  20, 10, 10,  0 },  // pending
  {   0,  0,  0,  0 },  // reserved
  {   2,  1,  1,  0 },  // apps
  {  -1,  1,  1,  0 },  // req granularity
  {   3,  0,  0,  0 },  // subqueues
 };
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // Will not preempt for over capacity queues
 verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appA)));
}

@Test
public void testPreemptCycle() {
 int[][] qData = new int[][]{
  //  /   A   B   C
  { 100, 40, 40, 20 },  // abs
  { 100, 100, 100, 100 },  // maxCap
  { 100,  0, 60, 40 },  // used
  {  10, 10,  0,  0 },  // pending
  {   0,  0,  0,  0 },  // reserved
  {   3,  1,  1,  1 },  // apps
  {  -1,  1,  1,  1 },  // req granularity
  {   3,  0,  0,  0 },  // subqueues
 };
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // ensure all pending rsrc from A get preempted from other queues
 verify(mDisp, times(10)).handle(argThat(new IsPreemptionRequestFor(appC)));
}

@Test
public void testOverCapacityImbalance() {
 int[][] qData = new int[][]{
  //  /   A   B   C
  { 100, 40, 40, 20 },  // abs
  { 100, 100, 100, 100 },  // maxCap
  { 100, 55, 45,  0 },  // used
  {  20, 10, 10,  0 },  // pending
  {   0,  0,  0,  0 },  // reserved
  {   2,  1,  1,  0 },  // apps
  {  -1,  1,  1,  0 },  // req granularity
  {   3,  0,  0,  0 },  // subqueues
 };
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // correct imbalance between over-capacity queues
 verify(mDisp, times(5)).handle(argThat(new IsPreemptionRequestFor(appA)));
}

@Test
public void testMaxCap() {
 int[][] qData = new int[][]{
   //  /   A   B   C
   { 100, 40, 40, 20 },  // abs
   { 100, 100, 45, 100 },  // maxCap
   { 100, 55, 45,  0 },  // used
   {  20, 10, 10,  0 },  // pending
   {   0,  0,  0,  0 },  // reserved
   {   2,  1,  1,  0 },  // apps
   {  -1,  1,  1,  0 },  // req granularity
   {   3,  0,  0,  0 },  // subqueues
  };
  ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
  policy.editSchedule();
  // despite the imbalance, since B is at maxCap, do not correct
  verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appA)));
}

@Test
public void testZeroGuar() {
 int[][] qData = new int[][] {
  //  /    A   B   C    D   E   F
   { 200, 100, 0, 99, 100, 10, 90 },  // abs
   { 200, 200, 200, 200, 200, 200, 200 },  // maxCap
   { 170,  80, 60, 20,  90, 90,  0 },  // used
   {  10,   0,  0,  0,  10,  0, 10 },  // pending
   {   0,   0,  0,  0,   0,  0,  0 },  // reserved
   {   4,   2,  1,  1,   2,  1,  1 },  // apps
   {  -1,  -1,  1,  1,  -1,  1,  1 },  // req granularity
   {   2,   2,  0,  0,   2,  0,  0 },  // subqueues
 };
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // verify capacity taken from A1, not B1 despite B1 being far over
 // its absolute guaranteed capacity
 verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appA)));
}

@Test
public void testHierarchical() {
 int[][] qData = new int[][] {
  //  /    A   B   C    D   E   F
  { 200, 100, 50, 50, 100, 10, 90 },  // abs
  { 200, 200, 200, 200, 200, 200, 200 },  // maxCap
  { 200, 110, 60, 50,  90, 90,  0 },  // used
  {  10,   0,  0,  0,  10,  0, 10 },  // pending
  {   0,   0,  0,  0,   0,  0,  0 },  // reserved
  {   4,   2,  1,  1,   2,  1,  1 },  // apps
  {  -1,  -1,  1,  1,  -1,  1,  1 },  // req granularity
  {   2,   2,  0,  0,   2,  0,  0 },  // subqueues
 };
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // verify capacity taken from A1, not B1 despite B1 being far over
 // its absolute guaranteed capacity
 verify(mDisp, times(10)).handle(argThat(new IsPreemptionRequestFor(appA)));
}

@Test
public void testNaturalTermination() {
 int[][] qData = new int[][]{
  //  /   A   B   C
  { 100, 40, 40, 20 },  // abs
  { 100, 100, 100, 100 },  // maxCap
  { 100, 55, 45,  0 },  // used
  {  20, 10, 10,  0 },  // pending
  {   0,  0,  0,  0 },  // reserved
  {   2,  1,  1,  0 },  // apps
  {  -1,  1,  1,  0 },  // req granularity
  {   3,  0,  0,  0 },  // subqueues
 };
 conf.setFloat(
   CapacitySchedulerConfiguration.PREEMPTION_NATURAL_TERMINATION_FACTOR,
   (float) 0.1);
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // ignore 10% imbalance between over-capacity queues
 verify(mDisp, never()).handle(isA(ContainerPreemptEvent.class));
}

@Test
public void testProportionalPreemption() {
 int[][] qData = new int[][]{
  //  /   A   B   C  D
  { 100, 10, 40, 20, 30 },  // abs
  { 100, 100, 100, 100, 100 },  // maxCap
  { 100, 30, 60, 10,  0 },  // used
  {  45, 20,  5, 20,  0 },  // pending
  {   0,  0,  0,  0,  0 },  // reserved
  {   3,  1,  1,  1,  0 },  // apps
  {  -1,  1,  1,  1,  1 },  // req granularity
  {   4,  0,  0,  0,  0 },  // subqueues
 };
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // A will preempt guaranteed-allocated.
 verify(mDisp, times(10)).handle(argThat(new IsPreemptionRequestFor(appA)));
}

@Test
public void testDeadzone() {
 int[][] qData = new int[][]{
  //  /   A   B   C
  { 100, 40, 40, 20 },  // abs
  { 100, 100, 100, 100 },  // maxCap
  { 100, 39, 43, 21 },  // used
  {  10, 10,  0,  0 },  // pending
  {   0,  0,  0,  0 },  // reserved
  {   3,  1,  1,  1 },  // apps
  {  -1,  1,  1,  1 },  // req granularity
  {   3,  0,  0,  0 },  // subqueues
 };
 conf.setFloat(
   CapacitySchedulerConfiguration.PREEMPTION_MAX_IGNORED_OVER_CAPACITY,
   (float) 0.1);
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // ignore 10% overcapacity to avoid jitter
 verify(mDisp, never()).handle(isA(ContainerPreemptEvent.class));
}

@Test
public void testDeadzone() {
 int[][] qData = new int[][]{
  //  /   A   B   C
  { 100, 40, 40, 20 },  // abs
  { 100, 100, 100, 100 },  // maxCap
  { 100, 39, 43, 21 },  // used
  {  10, 10,  0,  0 },  // pending
  {   0,  0,  0,  0 },  // reserved
  {   3,  1,  1,  1 },  // apps
  {  -1,  1,  1,  1 },  // req granularity
  {   3,  0,  0,  0 },  // subqueues
 };
 conf.setFloat(MAX_IGNORED_OVER_CAPACITY, (float) 0.1);
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // ignore 10% overcapacity to avoid jitter
 verify(mDisp, never()).handle(isA(ContainerPreemptEvent.class));
}

@Test
public void testZeroGuar() {
 int[][] qData = new int[][] {
  //  /    A   B   C    D   E   F
   { 200, 100, 0, 99, 100, 10, 90 },  // abs
   { 200, 200, 200, 200, 200, 200, 200 },  // maxCap
   { 170,  80, 60, 20,  90, 90,  0 },  // used
   {  10,   0,  0,  0,  10,  0, 10 },  // pending
   {   0,   0,  0,  0,   0,  0,  0 },  // reserved
   {   4,   2,  1,  1,   2,  1,  1 },  // apps
   {  -1,  -1,  1,  1,  -1,  1,  1 },  // req granularity
   {   2,   2,  0,  0,   2,  0,  0 },  // subqueues
 };
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // verify capacity taken from A1, not B1 despite B1 being far over
 // its absolute guaranteed capacity
 verify(mDisp, never()).handle(argThat(new IsPreemptionRequestFor(appA)));
}

@Test
public void testPerQueuePreemptionNotAllUntouchable() {
 int[][] qData = new int[][] {
  //  /      A                       E
  //               B     C     D           F     G     H
  { 2000, 1000,  800,  100,  100, 1000,  500,  300,  200 },  // abs
  { 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000 },  // maxCap
  { 2000, 1300,  300,  800,  200,  700,  500,    0,  200 },  // used
  {  300,    0,    0,    0,    0,  300,    0,  300,    0 },  // pending
  {    0,    0,    0,    0,    0,    0,    0,    0,    0 },  // reserved
  //             appA  appB  appC        appD  appE  appF
  {    6,    3,    1,    1,    1,    3,    1,    1,    1 },  // apps
  {   -1,   -1,    1,    1,    1,   -1,    1,    1,    1 },  // req granularity
  {    2,    3,    0,    0,    0,    3,    0,    0,    0 },  // subqueues
 };
 conf.setPreemptionDisabled("root.queueA.queueC", true);
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // Although queueC(appB) is way over capacity and is untouchable,
 // queueD(appC) is preemptable. Request should be filled from queueD(appC).
 verify(mDisp, times(100)).handle(argThat(new IsPreemptionRequestFor(appC)));
}

@Test
public void testPerQueuePreemptionNotAllUntouchable() {
 int[][] qData = new int[][] {
  //  /      A                       E
  //               B     C     D           F     G     H
  { 2000, 1000,  800,  100,  100, 1000,  500,  300,  200 },  // abs
  { 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000, 2000 },  // maxCap
  { 2000, 1300,  300,  800,  200,  700,  500,    0,  200 },  // used
  {  300,    0,    0,    0,    0,  300,    0,  300,    0 },  // pending
  {    0,    0,    0,    0,    0,    0,    0,    0,    0 },  // reserved
  //             appA  appB  appC        appD  appE  appF
  {    6,    3,    1,    1,    1,    3,    1,    1,    1 },  // apps
  {   -1,   -1,    1,    1,    1,   -1,    1,    1,    1 },  // req granularity
  {    2,    3,    0,    0,    0,    3,    0,    0,    0 },  // subqueues
 };
 schedConf.setPreemptionDisabled("root.queueA.queueC", true);
 ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
 policy.editSchedule();
 // Although queueC(appB) is way over capacity and is untouchable,
 // queueD(appC) is preemptable. Request should be filled from queueD(appC).
 verify(mDisp, times(100)).handle(argThat(new IsPreemptionRequestFor(appC)));
}

Popular methods of ProportionalCapacityPreemptionPolicy

cloneQueues
This method walks a tree of CSQueue and clones the portion of the state relevant for preemption in T
containerBasedPreemptOrKill
This method selects and tracks containers to be preemptionCandidates. If a container is in the targe
init
logToCSV
sortContainers
Compare by reversed priority order first, and then reversed containerId order
<init>
computeFixpointAllocation
Given a set of queues compute the fix-point distribution of unassigned resources among them. As pend
computeIdealResourceDistribution
This method computes (for a single level in the tree, passed as a List) the ideal assignment of reso
getContainersToPreempt
Based a resource preemption target drop reservations of containers and if necessary select container
getMostUnderservedQueues
getNonLabeledResources
This method returns all non labeled resources.
getResourceCalculator

Popular in Java

Reactive rest calls using spring rest template
setContentView (Activity)
getApplicationContext (Context)
getSupportFragmentManager (FragmentActivity)
MalformedURLException (java.net)
This exception is thrown when a program attempts to create an URL from an incorrect specification.
Arrays (java.util)
This class contains various methods for manipulating arrays (such as sorting and searching). This cl
Comparator (java.util)
A Comparator is used to compare two objects to determine their ordering with respect to each other.
HashMap (java.util)
HashMap is an implementation of Map. All optional operations are supported.All elements are permitte
Base64 (org.apache.commons.codec.binary)
Provides Base64 encoding and decoding as defined by RFC 2045.This class implements section 6.8. Base
Scheduler (org.quartz)
This is the main interface of a Quartz Scheduler. A Scheduler maintains a registry of org.quartz.Job
Best IntelliJ plugins

How to use editSchedulemethodin org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy

Best Java code snippets using org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy.editSchedule (Showing top 20 results out of 315)

How to use
editSchedule
method
in
org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy