private void sendEnoughNodeUpdatesToAssignFully() { for (RMNode node : rmNodes) { NodeUpdateSchedulerEvent nodeUpdateSchedulerEvent = new NodeUpdateSchedulerEvent(node); for (int i = 0; i < NODE_CAPACITY_MULTIPLE; i++) { scheduler.handle(nodeUpdateSchedulerEvent); } } }
private void addAppAttempt(ApplicationAttemptId attId) { AppAttemptAddedSchedulerEvent attempAddedEvent = new AppAttemptAddedSchedulerEvent(attId, false); scheduler.handle(attempAddedEvent); }
private void sendEnoughNodeUpdatesToAssignFully() { for (RMNode node : rmNodes) { NodeUpdateSchedulerEvent nodeUpdateSchedulerEvent = new NodeUpdateSchedulerEvent(node); for (int i = 0; i < NODE_CAPACITY_MULTIPLE; i++) { scheduler.handle(nodeUpdateSchedulerEvent); } } } }
/** * Add a node to the cluster and track the nodes in {@link #rmNodes}. * @param memory memory capacity of the node * @param cores cpu capacity of the node */ protected void addNode(int memory, int cores) { int id = rmNodes.size() + 1; RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(memory, cores), id, "127.0.0." + id); scheduler.handle(new NodeAddedSchedulerEvent(node)); rmNodes.add(node); } }
protected void createApplicationWithAMResource(ApplicationAttemptId attId, String queue, String user, Resource amResource) { RMContext rmContext = resourceManager.getRMContext(); RMApp rmApp = new RMAppImpl(attId.getApplicationId(), rmContext, conf, null, null, null, ApplicationSubmissionContext.newInstance(null, null, null, null, null, false, false, 0, amResource, null), null, null, 0, null, null, null); rmContext.getRMApps().put(attId.getApplicationId(), rmApp); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent( attId.getApplicationId(), queue, user); scheduler.handle(appAddedEvent); AppAttemptAddedSchedulerEvent attempAddedEvent = new AppAttemptAddedSchedulerEvent(attId, false); scheduler.handle(attempAddedEvent); }
@Test (timeout = 3000) public void testSchedulingUpdateOnNodeJoinLeave() throws InterruptedException { verifyNoCalls(); // Add one node String host = "127.0.0.1"; final int memory = 4096; final int cores = 4; RMNode node1 = MockNodes.newNodeInfo( 1, Resources.createResource(memory, cores), 1, host); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); long expectedCalls = 1; verifyExpectedCalls(expectedCalls, memory, cores); // Remove the node NodeRemovedSchedulerEvent nodeEvent2 = new NodeRemovedSchedulerEvent(node1); scheduler.handle(nodeEvent2); expectedCalls = 2; verifyExpectedCalls(expectedCalls, 0, 0); }
@Test public void testNoMoreCpuOnNode() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default", "user1", 2); FSAppAttempt app = scheduler.getSchedulerApp(attId); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); }
@Test public void testNoMoreCpuOnNode() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default", "user1", 2); FSAppAttempt app = scheduler.getSchedulerApp(attId); scheduler.update(); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); scheduler.handle(updateEvent); assertEquals(1, app.getLiveContainers().size()); }
public static FairScheduler setupFairScheduler(RMContext rmContext, Configuration conf, int numContainers) throws IOException { FairScheduler scheduler = new FairScheduler(); scheduler.setRMContext(rmContext); when(rmContext.getScheduler()).thenReturn(scheduler); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, rmContext); Resource resource = ReservationSystemTestUtil.calculateClusterResource(numContainers); RMNode node1 = MockNodes.newNodeInfo(1, resource, 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); return scheduler; }
private void addApplication(String queue, String user, ApplicationId appId) { RMAppEvent event = new RMAppEvent(appId, RMAppEventType.START); resourceManager.getRMContext().getRMApps().get(appId).handle(event); event = new RMAppEvent(appId, RMAppEventType.APP_NEW_SAVED); resourceManager.getRMContext().getRMApps().get(appId).handle(event); event = new RMAppEvent(appId, RMAppEventType.APP_ACCEPTED); resourceManager.getRMContext().getRMApps().get(appId).handle(event); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent( appId, queue, user); scheduler.handle(appAddedEvent); }
public static FairScheduler setupFairScheduler( ReservationSystemTestUtil testUtil, RMContext rmContext, Configuration conf, int numContainers) throws IOException { FairScheduler scheduler = new FairScheduler(); scheduler.setRMContext(rmContext); when(rmContext.getScheduler()).thenReturn(scheduler); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, rmContext); Resource resource = testUtil.calculateClusterResource(numContainers); RMNode node1 = MockNodes.newNodeInfo(1, resource, 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); return scheduler; }
private void testSchedulingAllowedToQueueZeroCapacityOfResource( String resource, int memory, int vCores) throws IOException { conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); generateAllocationFileWithZeroResource(resource); scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add a node RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2)); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); createSchedulingRequest(memory, vCores, "queueA", "user1", 1, 2); } }
private void registerNodeAndSubmitApp( int memory, int vcores, int appContainers, int appMemory) { RMNode node1 = MockNodes.newNodeInfo( 1, Resources.createResource(memory, vcores), 1, "node1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); assertEquals("Incorrect amount of resources in the cluster", memory, scheduler.rootMetrics.getAvailableMB()); assertEquals("Incorrect amount of resources in the cluster", vcores, scheduler.rootMetrics.getAvailableVirtualCores()); createSchedulingRequest(appMemory, "queueA", "user1", appContainers); scheduler.update(); // Sufficient node check-ins to fully schedule containers for (int i = 0; i < 3; i++) { NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1); scheduler.handle(nodeUpdate1); } assertEquals("app1's request is not met", memory - appContainers * appMemory, scheduler.rootMetrics.getAvailableMB()); }
@Test (expected = YarnException.class) public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); queueMgr.getLeafQueue("queue2", true); scheduler.getAllocationConfiguration().maxQueueResources.put("root.queue2", Resource.newInstance(1024, 1)); ApplicationAttemptId appAttId = createSchedulingRequest(1024, 1, "queue1", "user1", 3); RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2)); NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); scheduler.handle(nodeEvent); scheduler.handle(updateEvent); scheduler.handle(updateEvent); assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage()); scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); }
@Test (expected = YarnException.class) public void testMoveWouldViolateMaxResourcesConstraints() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); QueueManager queueMgr = scheduler.getQueueManager(); FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true); FSQueue queue2 = queueMgr.getLeafQueue("queue2", true); queue2.setMaxShare( new ConfigurableResource(Resource.newInstance(1024, 1))); ApplicationAttemptId appAttId = createSchedulingRequest(1024, 1, "queue1", "user1", 3); RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2)); NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); scheduler.handle(nodeEvent); scheduler.handle(updateEvent); scheduler.handle(updateEvent); assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage()); scheduler.moveApplication(appAttId.getApplicationId(), "queue2"); }
@Test public void testAggregateCapacityTracking() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // Add a node RMNode node1 = MockNodes .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.handle(nodeEvent1); assertEquals(1024, scheduler.getClusterResource().getMemory()); // Add another node RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(512), 2, "127.0.0.2"); NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2); scheduler.handle(nodeEvent2); assertEquals(1536, scheduler.getClusterResource().getMemory()); // Remove the first node NodeRemovedSchedulerEvent nodeEvent3 = new NodeRemovedSchedulerEvent(node1); scheduler.handle(nodeEvent3); assertEquals(512, scheduler.getClusterResource().getMemory()); }
@Test public void testEmptyQueueName() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // only default queue assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); // submit app with empty queue ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "", "user1"); scheduler.handle(appAddedEvent); // submission rejected assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); assertNull(scheduler.getSchedulerApp(appAttemptId)); assertEquals(0, resourceManager.getRMContext().getRMApps().size()); }
@Test public void testEmptyQueueName() throws Exception { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); // only default queue assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); // submit app with empty queue ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "", "user1"); scheduler.handle(appAddedEvent); // submission rejected assertEquals(1, scheduler.getQueueManager().getLeafQueues().size()); assertNull(scheduler.getSchedulerApp(appAttemptId)); assertEquals(0, resourceManager.getRMContext().getRMApps().size()); }
@Test public void testRemoveNodeUpdatesRootQueueMetrics() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024, 4), 1, "127.0.0.1"); NodeAddedSchedulerEvent addEvent = new NodeAddedSchedulerEvent(node1); scheduler.handle(addEvent); assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); scheduler.update(); // update shouldn't change things assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); NodeRemovedSchedulerEvent removeEvent = new NodeRemovedSchedulerEvent(node1); scheduler.handle(removeEvent); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); scheduler.update(); // update shouldn't change things assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); }
@Test public void testRemoveNodeUpdatesRootQueueMetrics() throws IOException { scheduler.init(conf); scheduler.start(); scheduler.reinitialize(conf, resourceManager.getRMContext()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024, 4), 1, "127.0.0.1"); NodeAddedSchedulerEvent addEvent = new NodeAddedSchedulerEvent(node1); scheduler.handle(addEvent); assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); scheduler.update(); // update shouldn't change things assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); NodeRemovedSchedulerEvent removeEvent = new NodeRemovedSchedulerEvent(node1); scheduler.handle(removeEvent); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); scheduler.update(); // update shouldn't change things assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB()); assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores()); }