private synchronized Configuration getConfiguration(Configuration conf, String... confFileNames) throws YarnException, IOException { for (String confFileName : confFileNames) { InputStream confFileInputStream = this.rmContext.getConfigurationProvider() .getConfigurationInputStream(conf, confFileName); if (confFileInputStream != null) { conf.addResource(confFileInputStream); } } return conf; }
private synchronized Configuration getConfiguration(Configuration conf, String... confFileNames) throws YarnException, IOException { for (String confFileName : confFileNames) { InputStream confFileInputStream = this.rmContext.getConfigurationProvider() .getConfigurationInputStream(conf, confFileName); if (confFileInputStream != null) { conf.addResource(confFileInputStream); } } return conf; }
private HostsFileReader createHostsFileReader(String includesFile, String excludesFile) throws IOException, YarnException { HostsFileReader hostsReader = new HostsFileReader(includesFile, (includesFile == null || includesFile.isEmpty()) ? null : this.rmContext.getConfigurationProvider() .getConfigurationInputStream(this.conf, includesFile), excludesFile, (excludesFile == null || excludesFile.isEmpty()) ? null : this.rmContext.getConfigurationProvider() .getConfigurationInputStream(this.conf, excludesFile)); return hostsReader; }
private HostsFileReader createHostsFileReader(String includesFile, String excludesFile) throws IOException, YarnException { HostsFileReader hostsReader = new HostsFileReader(includesFile, (includesFile == null || includesFile.isEmpty()) ? null : this.rmContext.getConfigurationProvider() .getConfigurationInputStream(this.conf, includesFile), excludesFile, (excludesFile == null || excludesFile.isEmpty()) ? null : this.rmContext.getConfigurationProvider() .getConfigurationInputStream(this.conf, excludesFile)); return hostsReader; }
private HostsFileReader createHostsFileReader(String includesFile, String excludesFile) throws IOException, YarnException { HostsFileReader hostsReader = new HostsFileReader(includesFile, (includesFile == null || includesFile.isEmpty()) ? null : this.rmContext.getConfigurationProvider() .getConfigurationInputStream(this.conf, includesFile), excludesFile, (excludesFile == null || excludesFile.isEmpty()) ? null : this.rmContext.getConfigurationProvider() .getConfigurationInputStream(this.conf, excludesFile)); return hostsReader; }
private synchronized Configuration getConfiguration(Configuration conf, String... confFileNames) throws YarnException, IOException { for (String confFileName : confFileNames) { InputStream confFileInputStream = this.rm.getRMContext().getConfigurationProvider() .getConfigurationInputStream(conf, confFileName); if (confFileInputStream != null) { conf.addResource(confFileInputStream); } } return conf; }
private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration( Configuration configuration) throws IOException { try { InputStream CSInputStream = this.rmContext.getConfigurationProvider() .getConfigurationInputStream(configuration, YarnConfiguration.CS_CONFIGURATION_FILE); if (CSInputStream != null) { configuration.addResource(CSInputStream); return new CapacitySchedulerConfiguration(configuration, false); } return new CapacitySchedulerConfiguration(configuration, true); } catch (Exception e) { throw new IOException(e); } }
private CapacitySchedulerConfiguration loadCapacitySchedulerConfiguration( Configuration configuration) throws IOException { try { InputStream CSInputStream = this.rmContext.getConfigurationProvider() .getConfigurationInputStream(configuration, YarnConfiguration.CS_CONFIGURATION_FILE); if (CSInputStream != null) { configuration.addResource(CSInputStream); return new CapacitySchedulerConfiguration(configuration, false); } return new CapacitySchedulerConfiguration(configuration, true); } catch (Exception e) { throw new IOException(e); } }
@Override public CapacitySchedulerConfiguration loadConfiguration(Configuration conf) throws IOException { try { InputStream csInputStream = this.rmContext.getConfigurationProvider() .getConfigurationInputStream(conf, YarnConfiguration.CS_CONFIGURATION_FILE); if (csInputStream != null) { conf.addResource(csInputStream); return new CapacitySchedulerConfiguration(conf, false); } return new CapacitySchedulerConfiguration(conf, true); } catch (Exception e) { throw new IOException(e); } } }
/** * Load DynamicResourceConfiguration from dynamic-resources.xml. * @param conf * @throws IOException */ public void loadDynamicResourceConfiguration(Configuration conf) throws IOException { try { // load dynamic-resources.xml InputStream drInputStream = this.rmContext.getConfigurationProvider() .getConfigurationInputStream(conf, YarnConfiguration.DR_CONFIGURATION_FILE); // write lock here on drConfig is unnecessary as here get called at // ResourceTrackerService get initiated and other read and write // operations haven't started yet. if (drInputStream != null) { this.drConf = new DynamicResourceConfiguration(conf, drInputStream); } else { this.drConf = new DynamicResourceConfiguration(conf); } } catch (Exception e) { throw new IOException(e); } }
public void refreshNodes(Configuration yarnConf) throws IOException, YarnException { synchronized (hostsReader) { if (null == yarnConf) { yarnConf = new YarnConfiguration(); } includesFile = yarnConf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH); excludesFile = yarnConf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH); hostsReader.updateFileNames(includesFile, excludesFile); hostsReader.refresh( includesFile.isEmpty() ? null : this.rmContext .getConfigurationProvider().getConfigurationInputStream( this.conf, includesFile), excludesFile.isEmpty() ? null : this.rmContext.getConfigurationProvider() .getConfigurationInputStream(this.conf, excludesFile)); printConfiguredHosts(); } for (NodeId nodeId: rmContext.getRMNodes().keySet()) { if (!isValidNode(nodeId.getHost())) { this.rmContext.getDispatcher().getEventHandler().handle( new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION)); } } }
public void refreshNodes(Configuration yarnConf) throws IOException, YarnException { synchronized (hostsReader) { if (null == yarnConf) { yarnConf = new YarnConfiguration(); } includesFile = yarnConf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH); excludesFile = yarnConf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH); hostsReader.updateFileNames(includesFile, excludesFile); hostsReader.refresh( includesFile.isEmpty() ? null : this.rmContext .getConfigurationProvider().getConfigurationInputStream( this.conf, includesFile), excludesFile.isEmpty() ? null : this.rmContext.getConfigurationProvider() .getConfigurationInputStream(this.conf, excludesFile)); printConfiguredHosts(); } for (NodeId nodeId: rmContext.getRMNodes().keySet()) { if (!isValidNode(nodeId.getHost())) { this.rmContext.getDispatcher().getEventHandler().handle( new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION)); } } }
@Override protected void serviceStart() throws Exception { super.serviceStart(); // ResourceTrackerServer authenticates NodeManager via Kerberos if // security is enabled, so no secretManager. Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); this.server = rpc.getServer( ResourceTracker.class, this, resourceTrackerAddress, conf, null, conf.getInt(YarnConfiguration.RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT)); // Enable service authorization? if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { InputStream inputStream = this.rmContext.getConfigurationProvider() .getConfigurationInputStream(conf, YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE); if (inputStream != null) { conf.addResource(inputStream); } refreshServiceAcls(conf, RMPolicyProvider.getInstance()); } this.server.start(); conf.updateConnectAddr(YarnConfiguration.RM_BIND_HOST, YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, server.getListenerAddress()); }
false)) { InputStream inputStream = this.rmContext.getConfigurationProvider() .getConfigurationInputStream(conf, YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
false)) { InputStream inputStream = this.rmContext.getConfigurationProvider() .getConfigurationInputStream(conf, YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
false)) { InputStream inputStream = this.rmContext.getConfigurationProvider() .getConfigurationInputStream(conf, YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
false)) { InputStream inputStream = this.rmContext.getConfigurationProvider() .getConfigurationInputStream(conf, YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
false)) { InputStream inputStream = this.rmContext.getConfigurationProvider() .getConfigurationInputStream(conf, YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
@Before public void setUp() throws Exception { resourceManager = new ResourceManager() { @Override protected RMNodeLabelsManager createNodeLabelManager() { RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(getConfig()); return mgr; } }; CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); resourceManager.init(conf); resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); mockContext = mock(RMContext.class); when(mockContext.getConfigurationProvider()).thenReturn( new LocalConfigurationProvider()); }
@Before public void setUp() throws Exception { resourceManager = new ResourceManager() { @Override protected RMNodeLabelsManager createNodeLabelManager() { RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(getConfig()); return mgr; } }; CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); resourceManager.init(conf); resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey(); resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey(); ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start(); mockContext = mock(RMContext.class); when(mockContext.getConfigurationProvider()).thenReturn( new LocalConfigurationProvider()); }