Resource clusterCapacity = Resource.newInstance(500 * 100 * 1024, 500 * 32); step = 1000L; ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); CapacityScheduler scheduler = testUtil.mockCapacityScheduler(500 * 100); String reservationQ = ReservationSystemTestUtil.getFullReservationQueueName(); float instConstraint = 100; float avgConstraint = 100; ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.createConf(reservationQ, timeWindow, instConstraint, avgConstraint); CapacityOverTimePolicy policy = new CapacityOverTimePolicy(); policy.init(reservationQ, conf); RMContext context = ReservationSystemTestUtil.createMockRMContext(); List<ReservationDefinition> list = new ArrayList<ReservationDefinition>(); for (long i = 0; i < numJobs; i++) { list.add(ReservationSystemTestUtil.generateRandomRR(rand, i)); ReservationSystemTestUtil.getNewReservationId(), "u" + i % 100, plan, list.get(i))) { acc++;
@Before public void setup() throws Exception { // 24h window timeWindow = 86400000L; // 1 sec step step = 1000L; // 25% avg cap on capacity avgConstraint = 25; // 70% instantaneous cap on capacity instConstraint = 70; initTime = System.currentTimeMillis(); minAlloc = Resource.newInstance(1024, 1); res = new DefaultResourceCalculator(); maxAlloc = Resource.newInstance(1024 * 8, 8); mAgent = mock(ReservationAgent.class); ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); QueueMetrics rootQueueMetrics = mock(QueueMetrics.class); String reservationQ = testUtil.getFullReservationQueueName(); Resource clusterResource = testUtil.calculateClusterResource(totCont); ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.createConf(reservationQ, timeWindow, instConstraint, avgConstraint); CapacityOverTimePolicy policy = new CapacityOverTimePolicy(); policy.init(reservationQ, conf); plan = new InMemoryPlan(rootQueueMetrics, policy, mAgent, clusterResource, step, res, minAlloc, maxAlloc, "dedicated", null, true); }
@Test public void testFairReservationSystemReinitialize() throws IOException { ReservationSystemTestUtil.setupFSAllocationFile(ALLOC_FILE); ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); RMContext mockRMContext = testUtil.createRMContext(conf); scheduler = ReservationSystemTestUtil.setupFairScheduler(testUtil, mockRMContext, conf, 10); final String planQNam = testUtil.getFullReservationQueueName(); ReservationSystemTestUtil.validateReservationQueue(reservationSystem, planQNam); ReservationSystemTestUtil.updateFSAllocationFile(ALLOC_FILE); scheduler.reinitialize(conf, mockRMContext); ReservationSystemTestUtil.validateNewReservationQueue (reservationSystem, newQueue);
private FairScheduler initializeFairScheduler() throws IOException { Configuration conf = createFSConfiguration(); ReservationSystemTestUtil.setupFSAllocationFile(ALLOC_FILE); // Setup mockRMContext = ReservationSystemTestUtil.createRMContext(conf); return ReservationSystemTestUtil .setupFairScheduler(mockRMContext, conf, 10); } }
private ReservationAllocation createReservationAllocation(long startTime, long deadline, long step, int[] alloc, ReservationId id, Resource minAlloc) { Map<ReservationInterval, Resource> allocations = ReservationSystemTestUtil .generateAllocation(startTime, step, alloc); ResourceCalculator rs = mock(ResourceCalculator.class); ReservationDefinition definition = ReservationSystemTestUtil .createSimpleReservationDefinition(startTime, deadline, step); return new InMemoryReservationAllocation(id, definition, "user", ReservationSystemTestUtil.reservationQ, startTime, startTime + step, allocations, rs, minAlloc, false); }
private void addFixedAllocation(long start, long step, int[] f) throws PlanningException { ReservationDefinition rDef = ReservationSystemTestUtil.createSimpleReservationDefinition( start, start + f.length * step, f.length * step); assertTrue(plan.toString(), plan.addReservation(new InMemoryReservationAllocation( ReservationSystemTestUtil.getNewReservationId(), rDef, "user_fixed", "dedicated", start, start + f.length * step, ReservationSystemTestUtil.generateAllocation(start, step, f), res, minAlloc), false)); }
@Test(expected = MismatchedUserException.class) public void testUserMismatch() throws IOException, PlanningException { // generate allocation from single tenant that exceed capacity int[] f = generateData(3600, (int) (0.5 * totCont)); ReservationId rid = ReservationSystemTestUtil.getNewReservationId(); plan.addReservation(new InMemoryReservationAllocation(rid, null, "u1", "dedicated", initTime, initTime + f.length, ReservationSystemTestUtil .generateAllocation(initTime, step, f), res, minAlloc)); // trying to update a reservation with a mismatching user plan.updateReservation(new InMemoryReservationAllocation(rid, null, "u2", "dedicated", initTime, initTime + f.length, ReservationSystemTestUtil .generateAllocation(initTime, step, f), res, minAlloc)); }
@Test public void testFairReservationSystemInitialize() throws IOException { ReservationSystemTestUtil.setupFSAllocationFile(ALLOC_FILE); ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); // Setup RMContext mockRMContext = testUtil.createRMContext(conf); scheduler = ReservationSystemTestUtil.setupFairScheduler(testUtil, mockRMContext, conf, 10); FairReservationSystem reservationSystem = new FairReservationSystem(); reservationSystem.setRMContext(mockRMContext); try { reservationSystem.reinitialize(scheduler.getConf(), mockRMContext); } catch (YarnException e) { Assert.fail(e.getMessage()); } ReservationSystemTestUtil.validateReservationQueue(reservationSystem, testUtil.getFullReservationQueueName()); }
@Test public void test() throws Exception { long period = Long.parseLong(recurrenceExpression); for (int i = 0; i < 1000; i++) { ReservationDefinition rr = createRandomRequest(i); if (rr != null) { ReservationId reservationID = ReservationSystemTestUtil.getNewReservationId(); try { agent.createReservation(reservationID, "u1", plan, rr); } catch (PlanningException p) { // happens } } } }
@Test public void testReinitialize() { ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); CapacityScheduler capScheduler = null; try { capScheduler = testUtil.mockCapacityScheduler(10); } catch (IOException e) { Assert.fail(e.getMessage()); String planQName = testUtil.getreservationQueueName(); ReservationSystemTestUtil.validateReservationQueue(reservationSystem, planQName); testUtil.updateQueueConfiguration(conf, newQ); try { capScheduler.reinitialize(conf, mockContext); Assert.fail(e.getMessage()); ReservationSystemTestUtil.validateNewReservationQueue(reservationSystem, newQ);
private CapacityScheduler initializeCapacityScheduler() { // stolen from TestCapacityScheduler CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); ReservationSystemTestUtil.setupQueueConfiguration(conf); CapacityScheduler cs = Mockito.spy(new CapacityScheduler()); cs.setConf(conf); mockRMContext = ReservationSystemTestUtil.createRMContext(conf); cs.setRMContext(mockRMContext); try { cs.serviceInit(conf); } catch (Exception e) { Assert.fail(e.getMessage()); } ReservationSystemTestUtil.initializeRMContext(10, cs, mockRMContext); return cs; }
@Before public void setup() throws Exception { long seed = rand.nextLong(); rand.setSeed(seed); LOG.info("Running with seed: " + seed); // setting completely loose quotas long timeWindow = 1000000L; Resource clusterCapacity = Resource.newInstance(numOfNodes * 1024, numOfNodes); step = 1000L; String reservationQ = ReservationSystemTestUtil.getFullReservationQueueName(); float instConstraint = 100; float avgConstraint = 100; ReservationSchedulerConfiguration conf = ReservationSystemTestUtil .createConf(reservationQ, timeWindow, instConstraint, avgConstraint); CapacityOverTimePolicy policy = new CapacityOverTimePolicy(); policy.init(reservationQ, conf); // setting conf to conf.setBoolean(GreedyReservationAgent.FAVOR_EARLY_ALLOCATION, allocateLeft); agent = (ReservationAgent) agentClass.newInstance(); agent.init(conf); QueueMetrics queueMetrics = mock(QueueMetrics.class); RMContext context = ReservationSystemTestUtil.createMockRMContext(); plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, resCalc, minAlloc, maxAlloc, "dedicated", null, true, context); }
@Test public void testInitialize() { ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); CapacityScheduler capScheduler = null; try { capScheduler = testUtil.mockCapacityScheduler(10); } catch (IOException e) { Assert.fail(e.getMessage()); } CapacityReservationSystem reservationSystem = new CapacityReservationSystem(); reservationSystem.setRMContext(capScheduler.getRMContext()); try { reservationSystem.reinitialize(capScheduler.getConf(), capScheduler.getRMContext()); } catch (YarnException e) { Assert.fail(e.getMessage()); } String planQName = testUtil.getreservationQueueName(); ReservationSystemTestUtil.validateReservationQueue(reservationSystem, planQName); }
@Before public void setUp() throws Exception { conf = createConfiguration(); ReservationSystemTestUtil.setupFSAllocationFile(ALLOC_FILE); ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); // Setup rmContext = TestUtils.getMockRMContext(); spyRMContext = spy(rmContext); fs = ReservationSystemTestUtil.setupFairScheduler(testUtil, spyRMContext, conf, 125); scheduler = fs; ConcurrentMap<ApplicationId, RMApp> spyApps = spy(new ConcurrentHashMap<ApplicationId, RMApp>()); RMApp rmApp = mock(RMApp.class); when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any())) .thenReturn(null); Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any()); when(spyRMContext.getRMApps()).thenReturn(spyApps); ReservationSystemTestUtil.setupFSAllocationFile(ALLOC_FILE); setupPlanFollower(); }
@Test public void testInitialize() throws IOException { try { reservationSystem.reinitialize(scheduler.getConfig(), rmContext); } catch (YarnException e) { Assert.fail(e.getMessage()); } if (getSchedulerType().equals(SchedulerType.CAPACITY)) { ReservationSystemTestUtil.validateReservationQueue(reservationSystem, ReservationSystemTestUtil.getReservationQueueName()); } else { ReservationSystemTestUtil.validateReservationQueue(reservationSystem, ReservationSystemTestUtil.getFullReservationQueueName()); } }
@Before public void setup() throws Exception { // 1 sec step step = 1000L; initTime = System.currentTimeMillis(); minAlloc = Resource.newInstance(1024, 1); res = new DefaultResourceCalculator(); maxAlloc = Resource.newInstance(1024 * 8, 8); mAgent = mock(ReservationAgent.class); ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); String reservationQ = testUtil.getFullReservationQueueName(); QueueMetrics rootQueueMetrics = mock(QueueMetrics.class); Resource clusterResource = testUtil.calculateClusterResource(totCont); ReservationSchedulerConfiguration conf = mock (ReservationSchedulerConfiguration.class); NoOverCommitPolicy policy = new NoOverCommitPolicy(); policy.init(reservationQ, conf); plan = new InMemoryPlan(rootQueueMetrics, policy, mAgent, clusterResource, step, res, minAlloc, maxAlloc, "dedicated", null, true); }
@Before public void setup() throws Exception { long seed = rand.nextLong(); rand.setSeed(seed); Log.info("Running with seed: " + seed); // setting completely loose quotas long timeWindow = 1000000L; Resource clusterCapacity = Resource.newInstance(100 * 1024, 100); step = 1000L; ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); String reservationQ = testUtil.getFullReservationQueueName(); float instConstraint = 100; float avgConstraint = 100; ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.createConf(reservationQ, timeWindow, instConstraint, avgConstraint); CapacityOverTimePolicy policy = new CapacityOverTimePolicy(); policy.init(reservationQ, conf); agent = new GreedyReservationAgent(); QueueMetrics queueMetrics = mock(QueueMetrics.class); plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", null, true); }
private void setupPlanFollower() throws Exception { ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); mClock = mock(Clock.class); mAgent = mock(ReservationAgent.class); String reservationQ = testUtil.getFullReservationQueueName(); AllocationConfiguration allocConf = fs.getAllocationConfiguration(); allocConf.setReservationWindow(20L); allocConf.setAverageCapacity(20); policy.init(reservationQ, allocConf); }
@Override public SharingPolicy getInitializedPolicy() { // 24h window long timeWindow = 86400000L; // 1 sec step long step = 1000L; // 25% avg cap on capacity float avgConstraint = 25; // 70% instantaneous cap on capacity float instConstraint = 75; String reservationQ = ReservationSystemTestUtil.getFullReservationQueueName(); conf = ReservationSystemTestUtil.createConf(reservationQ, timeWindow, instConstraint, avgConstraint); CapacityOverTimePolicy policy = new CapacityOverTimePolicy(); policy.init(reservationQ, conf); return policy; }
ReservationSystemTestUtil.createConf(plan.getQueueName(), 100000, instConstraint, avgConstraint); ReservationSystemTestUtil.getNewReservationId(); agent.createReservation(reservationID, "u3", plan, rr); ReservationSystemTestUtil.getNewReservationId(); agent.createReservation(reservationID2, "u3", plan, rr); ReservationSystemTestUtil.getNewReservationId(); try {