private void createYarnClient() { if (this.yarnConfAddress != null) { this.yarnConf.addResource(new Path(this.yarnConfAddress)); } if (this.hdfsConfAddress != null) { this.hdfsConf.addResource(new Path(this.hdfsConfAddress)); } int numRMConnectRetries = tonyConf.getInt(TonyConfigurationKeys.RM_CLIENT_CONNECT_RETRY_MULTIPLIER, TonyConfigurationKeys.DEFAULT_RM_CLIENT_CONNECT_RETRY_MULTIPLIER); long rmMaxWaitMS = yarnConf.getLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS) * numRMConnectRetries; yarnConf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, rmMaxWaitMS); if (System.getenv(Constants.HADOOP_CONF_DIR) != null) { hdfsConf.addResource(new Path(System.getenv(Constants.HADOOP_CONF_DIR) + File.separatorChar + Constants.CORE_SITE_CONF)); yarnConf.addResource(new Path(System.getenv(Constants.HADOOP_CONF_DIR) + File.separatorChar + Constants.CORE_SITE_CONF)); hdfsConf.addResource(new Path(System.getenv(Constants.HADOOP_CONF_DIR) + File.separatorChar + Constants.HDFS_SITE_CONF)); } yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConf); }
protected void configureFairScheduler(YarnConfiguration conf) throws IOException { // Disable queueMaxAMShare limitation for fair scheduler PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE)); out.println("<?xml version=\"1.0\"?>"); out.println("<allocations>"); out.println("<queueMaxAMShareDefault>-1.0</queueMaxAMShareDefault>"); out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>"); out.println("<queue name=\"root\">"); out.println(" <schedulingPolicy>drf</schedulingPolicy>"); out.println(" <weight>1.0</weight>"); out.println(" <fairSharePreemptionTimeout>100</fairSharePreemptionTimeout>"); out.println(" <minSharePreemptionTimeout>120</minSharePreemptionTimeout>"); out.println(" <fairSharePreemptionThreshold>.5</fairSharePreemptionThreshold>"); out.println("</queue>"); out.println("</allocations>"); out.close(); conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE); conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10); }
@Test(timeout=60000) public void testJobKillTimeout() throws Exception { long timeToWaitBeforeHardKill = 10000 + MRJobConfig.DEFAULT_MR_AM_HARD_KILL_TIMEOUT_MS; conf.setLong(MRJobConfig.MR_AM_HARD_KILL_TIMEOUT_MS, timeToWaitBeforeHardKill); clientDelegate = mock(ClientServiceDelegate.class); doAnswer( new Answer<ClientServiceDelegate>() { @Override public ClientServiceDelegate answer(InvocationOnMock invocation) throws Throwable { return clientDelegate; } } ).when(clientCache).getClient(any(JobID.class)); when(clientDelegate.getJobStatus(any(JobID.class))).thenReturn(new org.apache.hadoop.mapreduce.JobStatus(jobId, 0f, 0f, 0f, 0f, State.RUNNING, JobPriority.HIGH, "tmp", "tmp", "tmp", "tmp")); long startTimeMillis = System.currentTimeMillis(); yarnRunner.killJob(jobId); assertTrue("killJob should have waited at least " + timeToWaitBeforeHardKill + " ms.", System.currentTimeMillis() - startTimeMillis >= timeToWaitBeforeHardKill); }
/** * Prepare configuration for plugin tests. This method will also add the mini * DFS cluster's info to the configuration. * Note: the test program needs to setup the reader plugin by itself. * * @param conf * @param dfsCluster * @return the modified configuration */ public static YarnConfiguration prepareConfiguration(YarnConfiguration conf, MiniDFSCluster dfsCluster) { conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, dfsCluster.getURI().toString()); conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f); conf.setLong( YarnConfiguration.TIMELINE_SERVICE_ENTITYGROUP_FS_STORE_SCAN_INTERVAL_SECONDS, 1); conf.set(YarnConfiguration.TIMELINE_SERVICE_STORE, EntityGroupFSTimelineStore.class.getName()); return conf; }
@Before public void setup() throws UnknownHostException { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); conf = getConf(); UserGroupInformation.setConfiguration(conf); conf.set(YarnConfiguration.RECOVERY_ENABLED, "true"); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true); conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, 0); DefaultMetricsSystem.setMiniClusterMode(true); }
@Before public void setup() throws UnknownHostException { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); conf = getConf(); UserGroupInformation.setConfiguration(conf); conf.set(YarnConfiguration.RECOVERY_ENABLED, "true"); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true); conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, 0); DefaultMetricsSystem.setMiniClusterMode(true); }
@Test(timeout = 60000) public void testCompactionCycle() throws Exception { final DB mockdb = mock(DB.class); conf.setLong(YarnConfiguration.RM_LEVELDB_COMPACTION_INTERVAL_SECS, 1); stateStore = new LeveldbRMStateStore() { @Override protected DB openDatabase() throws Exception { return mockdb; } }; stateStore.init(conf); stateStore.start(); verify(mockdb, timeout(10000)).compactRange( (byte[]) isNull(), (byte[]) isNull()); }
@Override public RMStateStore getRMStateStore() throws Exception { YarnConfiguration conf = new YarnConfiguration(); conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI, workingDirPathURI.toString()); conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8); conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS, 900L); conf.setLong(YarnConfiguration.RM_EPOCH, epoch); conf.setLong(YarnConfiguration.RM_EPOCH_RANGE, getEpochRange()); if (adminCheckEnable) { conf.setBoolean( YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION, true); } this.store = new TestFileSystemRMStore(conf); Assert.assertEquals(store.getNumRetries(), 8); Assert.assertEquals(store.getRetryInterval(), 900L); Assert.assertTrue(store.fs.getConf() == store.fsConf); FileSystem previousFs = store.fs; store.startInternal(); Assert.assertTrue(store.fs != previousFs); Assert.assertTrue(store.fs.getConf() == store.fsConf); return store; }
@Test(timeout = 60000) public void testCompactionCycle() throws Exception { final DB mockdb = mock(DB.class); conf.setLong(YarnConfiguration.RM_LEVELDB_COMPACTION_INTERVAL_SECS, 1); LeveldbRMStateStore store = new LeveldbRMStateStore() { @Override protected DB openDatabase() throws Exception { return mockdb; } }; store.init(conf); store.start(); verify(mockdb, timeout(10000)).compactRange( (byte[]) isNull(), (byte[]) isNull()); store.close(); }
@Before public void setup() throws IOException { if (scheduler.equals(CapacityScheduler.class)) { // Since there is limited lifetime monitoring support in fair scheduler // it does not need queue setup long defaultLifetime = 15L; Configuration capacitySchedulerConfiguration = setUpCSQueue(maxLifetime, defaultLifetime); conf = new YarnConfiguration(capacitySchedulerConfiguration); } else { conf = new YarnConfiguration(); } // Always run for CS, since other scheduler do not support this. conf.setClass(YarnConfiguration.RM_SCHEDULER, scheduler, ResourceScheduler.class); Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); UserGroupInformation.setConfiguration(conf); conf.setLong(YarnConfiguration.RM_APPLICATION_MONITOR_INTERVAL_MS, 3000L); }
@Test (timeout = 20000) public void testZKClientRetry() throws Exception { TestZKClient zkClientTester = new TestZKClient(); final String path = "/test"; YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS); conf.setLong(YarnConfiguration.RM_ZK_RETRY_INTERVAL_MS, 100); final ZKRMStateStore store = (ZKRMStateStore) zkClientTester.getRMStateStore(conf); TestDispatcher dispatcher = new TestDispatcher(); store.setRMDispatcher(dispatcher); final AtomicBoolean assertionFailedInThread = new AtomicBoolean(false); stopServer(); Thread clientThread = new Thread() { @Override public void run() { try { store.getDataWithRetries(path, true); } catch (Exception e) { e.printStackTrace(); assertionFailedInThread.set(true); } } }; Thread.sleep(2000); startServer(); clientThread.join(); Assert.assertFalse(assertionFailedInThread.get()); }
@Test (timeout = 20000) public void testZKClientRetry() throws Exception { TestZKClient zkClientTester = new TestZKClient(); final String path = "/test"; YarnConfiguration conf = new YarnConfiguration(); conf.setInt(CommonConfigurationKeys.ZK_TIMEOUT_MS, ZK_TIMEOUT_MS); conf.setLong(CommonConfigurationKeys.ZK_RETRY_INTERVAL_MS, 100); final ZKRMStateStore store = (ZKRMStateStore) zkClientTester.getRMStateStore(conf); TestDispatcher dispatcher = new TestDispatcher(); store.setRMDispatcher(dispatcher); final AtomicBoolean assertionFailedInThread = new AtomicBoolean(false); testingServer.stop(); Thread clientThread = new Thread() { @Override public void run() { try { store.getData(path); } catch (Exception e) { e.printStackTrace(); assertionFailedInThread.set(true); } } }; Thread.sleep(2000); testingServer.start(); clientThread.join(); Assert.assertFalse(assertionFailedInThread.get()); }
@Test public void testWithNodeLabelUpdateEnabled() throws Exception { conf.setLong(YarnConfiguration.RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS, 1000); MockRM rm = new MockRM(conf); rm.init(conf); rm.getRMContext().getRMDelegatedNodeLabelsUpdater().nodeLabelsUpdateInterval = 3 * 1000; rm.start(); RMNodeLabelsManager mgr = rm.getRMContext().getNodeLabelManager(); mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); NodeId nodeId = toNodeId("h1:1234"); assertEquals(0, mgr.getLabelsOnNode(nodeId).size()); updateNodeLabels(nodeId, "x"); registerNode(rm, nodeId); Thread.sleep(4000); assertCollectionEquals(ImmutableSet.of("x"), mgr.getLabelsOnNode(nodeId)); // Ensure that node labels are updated if NodeLabelsProvider // gives different labels updateNodeLabels(nodeId, "y"); Thread.sleep(4000); assertCollectionEquals(ImmutableSet.of("y"), mgr.getLabelsOnNode(nodeId)); rm.stop(); }
YarnConfiguration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setLong(YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS, 0); createTimelineClient(conf); conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, newMaxRetries); conf.setLong(YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS, newIntervalMs); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@Test public void testWithNodeLabelUpdateDisabled() throws Exception { conf.setLong(YarnConfiguration.RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS, RMDelegatedNodeLabelsUpdater.DISABLE_DELEGATED_NODE_LABELS_UPDATE); MockRM rm = new MockRM(conf); rm.init(conf); rm.getRMContext().getRMDelegatedNodeLabelsUpdater().nodeLabelsUpdateInterval = 3 * 1000; rm.start(); RMNodeLabelsManager mgr = rm.getRMContext().getNodeLabelManager(); mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x")); NodeId nodeId = toNodeId("h1:1234"); updateNodeLabels(nodeId, "x"); registerNode(rm, nodeId); Thread.sleep(4000); // Ensure that even though timer is not run, node labels are fetched // when node is registered assertCollectionEquals(ImmutableSet.of("x"), mgr.getLabelsOnNode(nodeId)); rm.stop(); }
@Override public RMStateStore getRMStateStore() throws Exception { YarnConfiguration conf = new YarnConfiguration(); conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI, workingDirPathURI.toString()); conf.set(YarnConfiguration.FS_RM_STATE_STORE_RETRY_POLICY_SPEC, "100,6000"); conf.setInt(YarnConfiguration.FS_RM_STATE_STORE_NUM_RETRIES, 8); conf.setLong(YarnConfiguration.FS_RM_STATE_STORE_RETRY_INTERVAL_MS, 900L); if (adminCheckEnable) { conf.setBoolean( YarnConfiguration.YARN_INTERMEDIATE_DATA_ENCRYPTION, true); } this.store = new TestFileSystemRMStore(conf); Assert.assertEquals(store.getNumRetries(), 8); Assert.assertEquals(store.getRetryInterval(), 900L); return store; }
@Test(timeout = 20000) public void testAMfailedBetweenRMRestart() throws Exception { conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, 0); rm1 = new MockRM(conf); rm1.start();