final AtomicReference<AllocationConfiguration> allocConf = new AtomicReference<AllocationConfiguration>(); AllocationFileLoaderService allocsLoader = new AllocationFileLoaderService(); allocsLoader.init(conf); allocsLoader.setReloadListener(new AllocationFileLoaderService.Listener() {
public FairScheduler() { super(FairScheduler.class.getName()); context = new FSContext(this); allocsLoader = new AllocationFileLoaderService(); queueMgr = new QueueManager(this); maxRunningEnforcer = new MaxRunningAppsEnforcer(this); }
public FairScheduler() { super(FairScheduler.class.getName()); clock = new SystemClock(); allocsLoader = new AllocationFileLoaderService(); queueMgr = new QueueManager(this); maxRunningEnforcer = new MaxRunningAppsEnforcer(this); }
public FairScheduler() { super(FairScheduler.class.getName()); clock = new SystemClock(); allocsLoader = new AllocationFileLoaderService(); queueMgr = new QueueManager(this); maxRunningEnforcer = new MaxRunningAppsEnforcer(this); }
@Test public void testGetAllocationFileFromClasspath() { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, "test-fair-scheduler.xml"); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); File allocationFile = allocLoader.getAllocationFile(conf); assertEquals("test-fair-scheduler.xml", allocationFile.getName()); assertTrue(allocationFile.exists()); }
/** * Verify that you can't have the queue name with just a non breaking * whitespace in the allocations file. */ @Test (expected = AllocationConfigurationException.class) public void testQueueNameContainingNBWhitespace() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new OutputStreamWriter( new FileOutputStream(ALLOC_FILE), StandardCharsets.UTF_8)); out.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"); out.println("<allocations>"); out.println("<queue name=\"\u00a0\">"); out.println("</queue>"); out.println("</allocations>"); out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); }
@Test (expected = UnsupportedFileSystemException.class) public void testDenyGetAllocationFileFromUnsupportedFileSystem() throws UnsupportedFileSystemException { Configuration conf = new YarnConfiguration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, "badfs:///badfile"); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.getAllocationFile(conf); }
/** * Verify that you can't have the queue name with whitespace only in the * allocations file. */ @Test (expected = AllocationConfigurationException.class) public void testQueueNameContainingOnlyWhitespace() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("<queue name=\" \">"); out.println("</queue>"); out.println("</allocations>"); out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); }
/** * Verify that defaultQueueSchedulingMode can't accept FIFO as a value. */ @Test (expected = AllocationConfigurationException.class) public void testDefaultQueueSchedulingModeIsFIFO() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("<defaultQueueSchedulingPolicy>fifo</defaultQueueSchedulingPolicy>"); out.println("</allocations>"); out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); }
/** * Verify that you can't include periods as the queue name in the allocations * file. */ @Test (expected = AllocationConfigurationException.class) public void testQueueNameContainingPeriods() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("<queue name=\"parent1.child1\">"); out.println("</queue>"); out.println("</allocations>"); out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); }
/** * Verify that you can't include periods as the queue name in the allocations * file. */ @Test (expected = AllocationConfigurationException.class) public void testQueueNameContainingPeriods() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("<queue name=\"parent1.child1\">"); out.println("</queue>"); out.println("</allocations>"); out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); }
/** * Verify that you can't place queues at the same level as the root queue in * the allocations file. */ @Test (expected = AllocationConfigurationException.class) public void testQueueAlongsideRoot() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("<queue name=\"root\">"); out.println("</queue>"); out.println("<queue name=\"other\">"); out.println("</queue>"); out.println("</allocations>"); out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); }
/** * Verify that you can't place queues at the same level as the root queue in * the allocations file. */ @Test (expected = AllocationConfigurationException.class) public void testQueueAlongsideRoot() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("<queue name=\"root\">"); out.println("</queue>"); out.println("<queue name=\"other\">"); out.println("</queue>"); out.println("</allocations>"); out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); }
/** * Verify that you can't have dynamic user queue and reservable queue on * the same queue */ @Test (expected = AllocationConfigurationException.class) public void testReservableCannotBeCombinedWithDynamicUserQueue() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("<queue name=\"notboth\" type=\"parent\" >"); out.println("<reservation>"); out.println("</reservation>"); out.println("</queue>"); out.println("</allocations>"); out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); }
/** * Verify that you can't have dynamic user queue and reservable queue on * the same queue */ @Test (expected = AllocationConfigurationException.class) public void testReservableCannotBeCombinedWithDynamicUserQueue() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("<queue name=\"notboth\" type=\"parent\" >"); out.println("<reservation>"); out.println("</reservation>"); out.println("</queue>"); out.println("</allocations>"); out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); }
@Test public void testGetAllocationFileFromClasspath() { try { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, TEST_FAIRSCHED_XML); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); Path allocationFile = allocLoader.getAllocationFile(conf); assertEquals(TEST_FAIRSCHED_XML, allocationFile.getName()); assertTrue(fs.exists(allocationFile)); } catch (IOException e) { fail("Unable to access allocation file from classpath: " + e); } }
@Test public void testParentTagWithChild() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("<queue name=\"parent\" type=\"parent\">"); out.println(" <queue name=\"child\">"); out.println(" </queue>"); out.println("</queue>"); out.println("</allocations>"); out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); AllocationConfiguration queueConf = confHolder.allocConf; // Check whether queue 'parent' and 'child' are loaded successfully assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.PARENT) .contains("root.parent")); assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.LEAF) .contains("root.parent.child")); }
@Test public void testSimplePlacementPolicyFromConf() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); conf.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, false); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("</allocations>"); out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); AllocationConfiguration allocConf = confHolder.allocConf; QueuePlacementPolicy placementPolicy = allocConf.getPlacementPolicy(); List<QueuePlacementRule> rules = placementPolicy.getRules(); assertEquals(2, rules.size()); assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass()); assertEquals(false, rules.get(0).create); assertEquals(QueuePlacementRule.Default.class, rules.get(1).getClass()); }
@Test public void testSimplePlacementPolicyFromConf() throws Exception { Configuration conf = new Configuration(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE); conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false); conf.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, false); PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("</allocations>"); out.close(); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); allocLoader.init(conf); ReloadListener confHolder = new ReloadListener(); allocLoader.setReloadListener(confHolder); allocLoader.reloadAllocations(); AllocationConfiguration allocConf = confHolder.allocConf; QueuePlacementPolicy placementPolicy = allocConf.getPlacementPolicy(); List<QueuePlacementRule> rules = placementPolicy.getRules(); assertEquals(2, rules.size()); assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass()); assertEquals(false, rules.get(0).create); assertEquals(QueuePlacementRule.Default.class, rules.get(1).getClass()); }
@Test public void testGetAllocationFileFromFileSystem() throws IOException, URISyntaxException { Configuration conf = new YarnConfiguration(); File baseDir = new File(TEST_DIR + Path.SEPARATOR + "getAllocHDFS").getAbsoluteFile(); FileUtil.fullyDelete(baseDir); conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf); MiniDFSCluster hdfsCluster = builder.build(); String fsAllocPath = "hdfs://localhost:" + hdfsCluster.getNameNodePort() + Path.SEPARATOR + TEST_FAIRSCHED_XML; URL fschedURL = Thread.currentThread().getContextClassLoader() .getResource(TEST_FAIRSCHED_XML); FileSystem fs = FileSystem.get(conf); fs.copyFromLocalFile(new Path(fschedURL.toURI()), new Path(fsAllocPath)); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, fsAllocPath); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); Path allocationFile = allocLoader.getAllocationFile(conf); assertEquals(fsAllocPath, allocationFile.toString()); assertTrue(fs.exists(allocationFile)); hdfsCluster.shutdown(true); }