How to use

FSQueueMetrics

in

org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair

public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) {
 this.name = name;
 this.scheduler = scheduler;
 this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf());
 metrics.setMinShare(getMinShare());
 metrics.setMaxShare(getMaxShare());
 this.parent = parent;
}

/**
 * Dump scheduler state including states of all queues.
 */
private void dumpSchedulerState() {
 FSQueue rootQueue = queueMgr.getRootQueue();
 Resource clusterResource = getClusterResource();
 STATE_DUMP_LOG.debug(
   "FairScheduler state: Cluster Capacity: " + clusterResource +
   "  Allocations: " + rootMetrics.getAllocatedResources() +
   "  Availability: " + Resource.newInstance(
     rootMetrics.getAvailableMB(),
     rootMetrics.getAvailableVirtualCores()) +
   "  Demand: " + rootQueue.getDemand());
 STATE_DUMP_LOG.debug(rootQueue.dumpState());
}

public synchronized 
static FSQueueMetrics forQueue(String queueName, Queue parent,
  boolean enableUserMetrics, Configuration conf) {
 MetricsSystem ms = DefaultMetricsSystem.instance();
 QueueMetrics metrics = queueMetrics.get(queueName);
 if (metrics == null) {
  metrics = new FSQueueMetrics(ms, queueName, parent, enableUserMetrics, conf)
    .tag(QUEUE_INFO, queueName);
  
  // Register with the MetricsSystems
  if (ms != null) {
   metrics = ms.register(
       sourceName(queueName).toString(), 
       "Metrics for queue: " + queueName, metrics);
  }
  queueMetrics.put(queueName, metrics);
 }
 return (FSQueueMetrics)metrics;
}

/**
 * Subqueue metrics might be a little out of date because fair shares are
 * recalculated at the update interval, but the root queue metrics needs to
 * be updated synchronously with allocations and completions so that cluster
 * metrics will be consistent.
 */
private void updateRootQueueMetrics() {
 rootMetrics.setAvailableResourcesToQueue(
   Resources.subtract(
     clusterResource, rootMetrics.getAllocatedResources()));
}

public Resource getReservedResource() {
 reservedResource.setMemorySize(metrics.getReservedMB());
 reservedResource.setVirtualCores(metrics.getReservedVirtualCores());
 return reservedResource;
}

public QueueStatistics getQueueStatistics() {
 QueueStatistics stats =
   recordFactory.newRecordInstance(QueueStatistics.class);
 stats.setNumAppsSubmitted(getMetrics().getAppsSubmitted());
 stats.setNumAppsRunning(getMetrics().getAppsRunning());
 stats.setNumAppsPending(getMetrics().getAppsPending());
 stats.setNumAppsCompleted(getMetrics().getAppsCompleted());
 stats.setNumAppsKilled(getMetrics().getAppsKilled());
 stats.setNumAppsFailed(getMetrics().getAppsFailed());
 stats.setNumActiveUsers(getMetrics().getActiveUsers());
 stats.setAvailableMemoryMB(getMetrics().getAvailableMB());
 stats.setAllocatedMemoryMB(getMetrics().getAllocatedMB());
 stats.setPendingMemoryMB(getMetrics().getPendingMB());
 stats.setReservedMemoryMB(getMetrics().getReservedMB());
 stats.setAvailableVCores(getMetrics().getAvailableVirtualCores());
 stats.setAllocatedVCores(getMetrics().getAllocatedVirtualCores());
 stats.setPendingVCores(getMetrics().getPendingVirtualCores());
 stats.setReservedVCores(getMetrics().getReservedVirtualCores());
 stats.setAllocatedContainers(getMetrics().getAllocatedContainers());
 stats.setPendingContainers(getMetrics().getPendingContainers());
 stats.setReservedContainers(getMetrics().getReservedContainers());
 return stats;
}

public synchronized
static FSQueueMetrics forQueue(String queueName, Queue parent,
  boolean enableUserMetrics, Configuration conf) {
 MetricsSystem ms = DefaultMetricsSystem.instance();
 return forQueue(ms, queueName, parent, enableUserMetrics, conf);
}

/**
 * Check if preemption is enabled and the utilization threshold for
 * preemption is met.
 *
 * @return true if preemption should be attempted, false otherwise.
 */
private boolean shouldAttemptPreemption() {
 if (preemptionEnabled) {
  return (preemptionUtilizationThreshold < Math.max(
    (float) rootMetrics.getAllocatedMB() / clusterResource.getMemory(),
    (float) rootMetrics.getAllocatedVirtualCores() /
      clusterResource.getVirtualCores()));
 }
 return false;
}

private void registerNodeAndSubmitApp(
  int memory, int vcores, int appContainers, int appMemory) {
 RMNode node1 = MockNodes.newNodeInfo(
   1, Resources.createResource(memory, vcores), 1, "node1");
 NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
 scheduler.handle(nodeEvent1);
 assertEquals("Incorrect amount of resources in the cluster",
   memory, scheduler.rootMetrics.getAvailableMB());
 assertEquals("Incorrect amount of resources in the cluster",
   vcores, scheduler.rootMetrics.getAvailableVirtualCores());
 createSchedulingRequest(appMemory, "queueA", "user1", appContainers);
 scheduler.update();
 // Sufficient node check-ins to fully schedule containers
 for (int i = 0; i < 3; i++) {
  NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
  scheduler.handle(nodeUpdate1);
 }
 assertEquals("app1's request is not met",
   memory - appContainers * appMemory,
   scheduler.rootMetrics.getAvailableMB());
}

/**
 * Test if the metric scheduling policy is set correctly.
 */
@Test
public void testSchedulingPolicy() {
 String queueName = "single";
 FSQueueMetrics metrics = FSQueueMetrics.forQueue(ms, queueName, null, false,
   CONF);
 metrics.setSchedulingPolicy("drf");
 checkSchedulingPolicy(queueName, "drf");
 // test resetting the scheduling policy
 metrics.setSchedulingPolicy("fair");
 checkSchedulingPolicy(queueName, "fair");
}

@Override
public void setFairShare(Resource fairShare) {
 this.fairShare = fairShare;
 metrics.setFairShare(fairShare);
}

assertEquals(1024, scheduler.rootMetrics.getReservedMB());
assertEquals(1024, scheduler.rootMetrics.getReservedMB());
assertEquals(0, scheduler.rootMetrics.getReservedMB());

@Before
public void setUp() throws Exception {
 conf = new FairSchedulerConfiguration();
 FairScheduler scheduler = mock(FairScheduler.class);
 AllocationConfiguration allocConf = new AllocationConfiguration(conf);
 when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
 when(scheduler.getConf()).thenReturn(conf);
 SystemClock clock = new SystemClock();
 when(scheduler.getClock()).thenReturn(clock);
 notEmptyQueues = new HashSet<FSQueue>();
 queueManager = new QueueManager(scheduler) {
  @Override
  public boolean isEmpty(FSQueue queue) {
   return !notEmptyQueues.contains(queue);
  }
 };
 FSQueueMetrics.forQueue("root", null, true, conf);
 queueManager.initialize(conf);
}

/**
 * Subqueue metrics might be a little out of date because fair shares are
 * recalculated at the update interval, but the root queue metrics needs to
 * be updated synchronously with allocations and completions so that cluster
 * metrics will be consistent.
 */
private void updateRootQueueMetrics() {
 rootMetrics.setAvailableResourcesToQueue(
   Resources.subtract(
     clusterResource, rootMetrics.getAllocatedResources()));
}

/**
 * Check if preemption is enabled and the utilization threshold for
 * preemption is met.
 *
 * @return true if preemption should be attempted, false otherwise.
 */
private boolean shouldAttemptPreemption() {
 if (preemptionEnabled) {
  return (preemptionUtilizationThreshold < Math.max(
    (float) rootMetrics.getAllocatedMB() / clusterResource.getMemory(),
    (float) rootMetrics.getAllocatedVirtualCores() /
      clusterResource.getVirtualCores()));
 }
 return false;
}

@Override
public void setFairShare(Resource fairShare) {
 this.fairShare = fairShare;
 metrics.setFairShare(fairShare);
}

assertEquals(1024, scheduler.rootMetrics.getReservedMB());
assertEquals(1024, scheduler.rootMetrics.getReservedMB());
assertEquals(0, scheduler.rootMetrics.getReservedMB());

/**
 * Recompute the internal variables used by the scheduler - per-job weights,
 * fair shares, deficits, minimum slot allocations, and amount of used and
 * required resources per job.
 */
protected synchronized void update() {
 long start = getClock().getTime();
 updateStarvationStats(); // Determine if any queues merit preemption
 FSQueue rootQueue = queueMgr.getRootQueue();
 // Recursively update demands for all queues
 rootQueue.updateDemand();
 rootQueue.setFairShare(clusterResource);
 // Recursively compute fair shares for all queues
 // and update metrics
 rootQueue.recomputeShares();
 updateRootQueueMetrics();
 if (LOG.isDebugEnabled()) {
  if (--updatesToSkipForDebug < 0) {
   updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY;
   LOG.debug("Cluster Capacity: " + clusterResource +
     "  Allocations: " + rootMetrics.getAllocatedResources() +
     "  Availability: " + Resource.newInstance(
     rootMetrics.getAvailableMB(),
     rootMetrics.getAvailableVirtualCores()) +
     "  Demand: " + rootQueue.getDemand());
  }
 }
 long duration = getClock().getTime() - start;
 fsOpDurations.addUpdateCallDuration(duration);
}

public synchronized 
static FSQueueMetrics forQueue(String queueName, Queue parent,
  boolean enableUserMetrics, Configuration conf) {
 MetricsSystem ms = DefaultMetricsSystem.instance();
 QueueMetrics metrics = queueMetrics.get(queueName);
 if (metrics == null) {
  metrics = new FSQueueMetrics(ms, queueName, parent, enableUserMetrics, conf)
    .tag(QUEUE_INFO, queueName);
  
  // Register with the MetricsSystems
  if (ms != null) {
   metrics = ms.register(
       sourceName(queueName).toString(), 
       "Metrics for queue: " + queueName, metrics);
  }
  queueMetrics.put(queueName, metrics);
 }
 return (FSQueueMetrics)metrics;
}

public FSQueue(String name, FairScheduler scheduler, FSParentQueue parent) {
 this.name = name;
 this.scheduler = scheduler;
 this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf());
 metrics.setMinShare(getMinShare());
 metrics.setMaxShare(getMaxShare());
 this.parent = parent;
}