@BeforeClass public static void setup() { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768); YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo"); startYARNWithConfig(YARN_CONFIGURATION); }
@BeforeClass public static void setup() throws Exception { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.set("yarn.scheduler.capacity.root.queues", "default,qa-team"); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.default.capacity", 40); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.qa-team.capacity", 60); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-capacityscheduler"); startYARNWithConfig(YARN_CONFIGURATION); restClientExecutor = Executors.newSingleThreadExecutor(); restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), restClientExecutor); }
@BeforeClass public static void setup() throws Exception { zkServer = new TestingServer(); storageDir = FOLDER.newFolder().getAbsolutePath(); // startYARNWithConfig should be implemented by subclass YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, LOG_DIR); YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 4096); startYARNWithConfig(YARN_CONFIGURATION); }
public YarnConfiguration createRegistryConfiguration() { YarnConfiguration conf = new YarnConfiguration(); conf.setInt(RegistryConstants.KEY_REGISTRY_ZK_CONNECTION_TIMEOUT, 1000); conf.setInt(RegistryConstants.KEY_REGISTRY_ZK_RETRY_INTERVAL, 500); conf.setInt(RegistryConstants.KEY_REGISTRY_ZK_RETRY_TIMES, 10); conf.setInt(RegistryConstants.KEY_REGISTRY_ZK_RETRY_CEILING, 10); conf.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, zookeeper.getConnectionString()); return conf; } }
public YarnConfiguration createRegistryConfiguration() { YarnConfiguration conf = new YarnConfiguration(); conf.setInt(RegistryConstants.KEY_REGISTRY_ZK_CONNECTION_TIMEOUT, 1000); conf.setInt(RegistryConstants.KEY_REGISTRY_ZK_RETRY_INTERVAL, 500); conf.setInt(RegistryConstants.KEY_REGISTRY_ZK_RETRY_TIMES, 10); conf.setInt(RegistryConstants.KEY_REGISTRY_ZK_RETRY_CEILING, 10); conf.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, zookeeper.getConnectionString()); return conf; } }
private static Configuration createConfForDelegationTokenNodeSplit( int splitIndex) { YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.ZK_DELEGATION_TOKEN_NODE_SPLIT_INDEX, splitIndex); return conf; }
private static Configuration createConfForAppNodeSplit(int splitIndex) { YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.ZK_APPID_NODE_SPLIT_INDEX, splitIndex); return conf; }
@Test(timeout = 60000) public void testNonDefaultMinimumAllocation() throws Exception { // Set custom min-alloc to test tweaking it int allocMB = 1536; YarnConfiguration conf = new YarnConfiguration(TestFifoScheduler.conf); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, allocMB); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, allocMB * 10); // Test for something lesser than this. testMinimumAllocation(conf, allocMB / 2); }
@Test public void testNonDefaultMinimumAllocation() throws Exception { // Set custom min-alloc to test tweaking it int allocMB = 1536; YarnConfiguration conf = new YarnConfiguration(TestFifoScheduler.conf); conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, allocMB); conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, allocMB * 10); // Test for something lesser than this. testMinimumAllocation(conf, allocMB / 2); }
@Test public void testZKAuths() throws Exception { TestZKClient zkClientTester = new TestZKClient(); YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.RM_ZK_NUM_RETRIES, 1); conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS); conf.set(YarnConfiguration.RM_ZK_ACL, TEST_ACL); conf.set(YarnConfiguration.RM_ZK_AUTH, TEST_AUTH_GOOD); zkClientTester.getRMStateStore(conf); }
@Before public void setup() throws UnknownHostException { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); conf = new YarnConfiguration(); UserGroupInformation.setConfiguration(conf); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); }
@Test public void testZKAuths() throws Exception { TestZKClient zkClientTester = new TestZKClient(); YarnConfiguration conf = new YarnConfiguration(); conf.setInt(CommonConfigurationKeys.ZK_NUM_RETRIES, 1); conf.setInt(CommonConfigurationKeys.ZK_TIMEOUT_MS, ZK_TIMEOUT_MS); conf.set(CommonConfigurationKeys.ZK_ACL, TEST_ACL); conf.set(CommonConfigurationKeys.ZK_AUTH, TEST_AUTH_GOOD); zkClientTester.getRMStateStore(conf); } }
@Before public void setup() throws UnknownHostException { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); conf = new YarnConfiguration(); UserGroupInformation.setConfiguration(conf); conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS); }
@Test(timeout=10000) public void testDispatchStopOnTimeout() throws Exception { BlockingQueue<Event> eventQueue = new LinkedBlockingQueue<Event>(); eventQueue = spy(eventQueue); // simulate dispatcher is not drained. when(eventQueue.isEmpty()).thenReturn(false); YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 2000); DrainDispatcher disp = new DrainDispatcher(eventQueue); disp.init(conf); disp.setDrainEventsOnStop(); disp.start(); disp.waitForEventThreadToWait(); disp.close(); } }
@Test(timeout=10000) public void testDispatchStopOnTimeout() throws Exception { BlockingQueue<Event> eventQueue = new LinkedBlockingQueue<Event>(); eventQueue = spy(eventQueue); // simulate dispatcher is not drained. when(eventQueue.isEmpty()).thenReturn(false); YarnConfiguration conf = new YarnConfiguration(); conf.setInt(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 2000); DrainDispatcher disp = new DrainDispatcher(eventQueue); disp.init(conf); disp.setDrainEventsOnStop(); disp.start(); disp.waitForEventThreadToWait(); disp.close(); }
@Before public void setup() { conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); conf.setInt(YarnConfiguration.NUMBER_OF_ASYNC_ENTITIES_TO_MERGE, 3); if (!currTestName.getMethodName() .contains("testRetryOnConnectionFailure")) { client = createTimelineClient(conf); } }
@Before public void createAndStartRM() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); YarnConfiguration conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); conf.setBoolean( YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true); conf.setInt( YarnConfiguration.NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS, 100); startRM(conf); }
public void createAndStartRMWithAutoUpdateContainer() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); YarnConfiguration conf = new YarnConfiguration(csConf); conf.setBoolean(YarnConfiguration.RM_AUTO_UPDATE_CONTAINERS, true); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); conf.setBoolean( YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true); conf.setInt( YarnConfiguration.NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS, 100); startRM(conf); }
@Before public void createAndStartRM() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); YarnConfiguration conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER, YarnConfiguration.PROCESSOR_RM_PLACEMENT_CONSTRAINTS_HANDLER); conf.setInt( YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS, 1); startRM(conf); }