/** * Return if the specified tasktracker is in the hosts list, * if one was configured. If none was configured, then this * returns true. */ private boolean inHostsList(TaskTrackerStatus status) { Set<String> hostsList = hostsReader.getHosts(); return (hostsList.isEmpty() || hostsList.contains(status.getHost())); }
/** * Return if the specified tasktracker is in the hosts list, * if one was configured. If none was configured, then this * returns true. */ private boolean inHostsList(TaskTrackerStatus status) { Set<String> hostsList = hostsReader.getHosts(); return (hostsList.isEmpty() || hostsList.contains(status.getHost())); }
/** * Return if the specified tasktracker is in the hosts list, * if one was configured. If none was configured, then this * returns true. */ private boolean inHostsList(TaskTrackerStatus status) { Set<String> hostsList = hostsReader.getHosts(); return (hostsList.isEmpty() || hostsList.contains(status.getHost())); }
private void printConfiguredHosts() { if (!LOG.isDebugEnabled()) { return; } LOG.debug("hostsReader: in=" + conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH) + " out=" + conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH)); for (String include : hostsReader.getHosts()) { LOG.debug("include: " + include); } for (String exclude : hostsReader.getExcludedHosts()) { LOG.debug("exclude: " + exclude); } }
private void printConfiguredHosts() { if (!LOG.isDebugEnabled()) { return; } LOG.debug("hostsReader: in=" + conf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH) + " out=" + conf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH)); for (String include : hostsReader.getHosts()) { LOG.debug("include: " + include); } for (String exclude : hostsReader.getExcludedHosts()) { LOG.debug("exclude: " + exclude); } }
public boolean isValidNode(String hostName) { synchronized (hostsReader) { Set<String> hostsList = hostsReader.getHosts(); Set<String> excludeList = hostsReader.getExcludedHosts(); String ip = NetUtils.normalizeHostName(hostName); return (hostsList.isEmpty() || hostsList.contains(hostName) || hostsList .contains(ip)) && !(excludeList.contains(hostName) || excludeList.contains(ip)); } }
public boolean isValidNode(String hostName) { synchronized (hostsReader) { Set<String> hostsList = hostsReader.getHosts(); Set<String> excludeList = hostsReader.getExcludedHosts(); String ip = NetUtils.normalizeHostName(hostName); return (hostsList.isEmpty() || hostsList.contains(hostName) || hostsList .contains(ip)) && !(excludeList.contains(hostName) || excludeList.contains(ip)); } }
/** * Keeps track of which datanodes/ipaddress are allowed to connect to the namenode. */ private boolean inHostsList(DatanodeID node, String ipAddr) { Set<String> hostsList = hostsReader.getHosts(); return (hostsList.isEmpty() || (ipAddr != null && hostsList.contains(ipAddr)) || hostsList.contains(node.getHost()) || hostsList.contains(node.getName()) || ((node instanceof DatanodeInfo) && hostsList.contains(((DatanodeInfo)node).getHostName()))); }
/** * Returns a set of dead nodes. (nodes that are expected to be alive) */ public Collection<String> getDeadNodes() { List<String> activeHosts = new ArrayList<String>(); synchronized(taskTrackers) { for (TaskTracker tt : taskTrackers.values()) { activeHosts.add(tt.getStatus().getHost()); } } // dead hosts are the difference between active and known hosts // We don't consider a blacklisted host to be dead. Set<String> knownHosts = new HashSet<String>(hostsReader.getHosts()); knownHosts.removeAll(activeHosts); // Also remove the excluded nodes as getHosts() returns them as well knownHosts.removeAll(hostsReader.getExcludedHosts()); Set<String> deadHosts = knownHosts; return deadHosts; }
@Test public void testHostFileReaderWithCommentsOnly() throws Exception { FileWriter efw = new FileWriter(excludesFile); FileWriter ifw = new FileWriter(includesFile); efw.write("#DFS-Hosts-excluded\n"); efw.close(); ifw.write("#Hosts-in-DFS\n"); ifw.close(); HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); int includesLen = hfp.getHosts().size(); int excludesLen = hfp.getExcludedHosts().size(); assertEquals(0, includesLen); assertEquals(0, excludesLen); assertFalse(hfp.getHosts().contains("somehost5")); assertFalse(hfp.getExcludedHosts().contains("somehost5")); }
/** * Keeps track of which datanodes/ipaddress are allowed to connect to the namenode. */ private boolean inHostsList(DatanodeID node, String ipAddr) { Set<String> hostsList = hostsReader.getHosts(); return (hostsList.isEmpty() || (ipAddr != null && hostsList.contains(ipAddr)) || hostsList.contains(node.getHost()) || hostsList.contains(node.getName()) || ((node instanceof DatanodeInfo) && ( hostsList.contains(((DatanodeInfo) node).getHostName()) || hostsList.contains(((DatanodeInfo) node).getHostName() + ":" + node.getPort())))) ; }
@Test public void testHostFileReaderWithNull() throws Exception { FileWriter efw = new FileWriter(excludesFile); FileWriter ifw = new FileWriter(includesFile); efw.close(); ifw.close(); HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); int includesLen = hfp.getHosts().size(); int excludesLen = hfp.getExcludedHosts().size(); // TestCase1: Check if lines beginning with # are ignored assertEquals(0, includesLen); assertEquals(0, excludesLen); // TestCase2: Check if given host names are reported by getHosts and // getExcludedHosts assertFalse(hfp.getHosts().contains("somehost5")); assertFalse(hfp.getExcludedHosts().contains("somehost5")); }
@Test public void testHostFileReaderWithSpaces() throws Exception { FileWriter efw = new FileWriter(excludesFile); FileWriter ifw = new FileWriter(includesFile); efw.write("#DFS-Hosts-excluded\n"); efw.write(" somehost somehost2"); efw.write(" somehost3 # somehost4"); efw.close(); ifw.write("#Hosts-in-DFS\n"); ifw.write(" somehost somehost2"); ifw.write(" somehost3 # somehost4"); ifw.close(); HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); int includesLen = hfp.getHosts().size(); int excludesLen = hfp.getExcludedHosts().size(); assertEquals(3, includesLen); assertEquals(3, excludesLen); assertTrue(hfp.getHosts().contains("somehost3")); assertFalse(hfp.getHosts().contains("somehost5")); assertFalse(hfp.getHosts().contains("somehost4")); assertTrue(hfp.getExcludedHosts().contains("somehost3")); assertFalse(hfp.getExcludedHosts().contains("somehost5")); assertFalse(hfp.getExcludedHosts().contains("somehost4")); }
private void refreshHostsReader( Configuration yarnConf, boolean graceful, Integer timeout) throws IOException, YarnException { // resolve the default timeout to the decommission timeout that is // configured at this moment if (null == timeout) { timeout = readDecommissioningTimeout(yarnConf); } if (null == yarnConf) { yarnConf = new YarnConfiguration(); } includesFile = yarnConf.get(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_INCLUDE_FILE_PATH); excludesFile = yarnConf.get(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, YarnConfiguration.DEFAULT_RM_NODES_EXCLUDE_FILE_PATH); LOG.info("refreshNodes excludesFile " + excludesFile); hostsReader.refresh(includesFile, excludesFile); printConfiguredHosts(); LOG.info("hostsReader include:{" + StringUtils.join(",", hostsReader.getHosts()) + "} exclude:{" + StringUtils.join(",", hostsReader.getExcludedHosts()) + "}"); handleExcludeNodeList(graceful, timeout); }
@Test public void testHostFileReaderWithCommentsOnly() throws Exception { FileWriter efw = new FileWriter(excludesFile); FileWriter ifw = new FileWriter(includesFile); efw.write("#DFS-Hosts-excluded\n"); efw.close(); ifw.write("#Hosts-in-DFS\n"); ifw.close(); HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); int includesLen = hfp.getHosts().size(); int excludesLen = hfp.getExcludedHosts().size(); assertEquals(0, includesLen); assertEquals(0, excludesLen); assertFalse(hfp.getHosts().contains("somehost5")); assertFalse(hfp.getExcludedHosts().contains("somehost5")); }
@Test public void testHostFileReaderWithCommentsOnly() throws Exception { FileWriter efw = new FileWriter(excludesFile); FileWriter ifw = new FileWriter(includesFile); efw.write("#DFS-Hosts-excluded\n"); efw.close(); ifw.write("#Hosts-in-DFS\n"); ifw.close(); HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); int includesLen = hfp.getHosts().size(); int excludesLen = hfp.getExcludedHosts().size(); assertEquals(0, includesLen); assertEquals(0, excludesLen); assertFalse(hfp.getHosts().contains("somehost5")); assertFalse(hfp.getExcludedHosts().contains("somehost5")); }
@Test public void testHostFileReaderWithNull() throws Exception { FileWriter efw = new FileWriter(excludesFile); FileWriter ifw = new FileWriter(includesFile); efw.close(); ifw.close(); HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); int includesLen = hfp.getHosts().size(); int excludesLen = hfp.getExcludedHosts().size(); // TestCase1: Check if lines beginning with # are ignored assertEquals(0, includesLen); assertEquals(0, excludesLen); // TestCase2: Check if given host names are reported by getHosts and // getExcludedHosts assertFalse(hfp.getHosts().contains("somehost5")); assertFalse(hfp.getExcludedHosts().contains("somehost5")); }
@Test public void testHostFileReaderWithNull() throws Exception { FileWriter efw = new FileWriter(excludesFile); FileWriter ifw = new FileWriter(includesFile); efw.close(); ifw.close(); HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); int includesLen = hfp.getHosts().size(); int excludesLen = hfp.getExcludedHosts().size(); // TestCase1: Check if lines beginning with # are ignored assertEquals(0, includesLen); assertEquals(0, excludesLen); // TestCase2: Check if given host names are reported by getHosts and // getExcludedHosts assertFalse(hfp.getHosts().contains("somehost5")); assertFalse(hfp.getExcludedHosts().contains("somehost5")); }
@Test public void testHostFileReaderWithSpaces() throws Exception { FileWriter efw = new FileWriter(excludesFile); FileWriter ifw = new FileWriter(includesFile); efw.write("#DFS-Hosts-excluded\n"); efw.write(" somehost somehost2"); efw.write(" somehost3 # somehost4"); efw.close(); ifw.write("#Hosts-in-DFS\n"); ifw.write(" somehost somehost2"); ifw.write(" somehost3 # somehost4"); ifw.close(); HostsFileReader hfp = new HostsFileReader(includesFile, excludesFile); int includesLen = hfp.getHosts().size(); int excludesLen = hfp.getExcludedHosts().size(); assertEquals(3, includesLen); assertEquals(3, excludesLen); assertTrue(hfp.getHosts().contains("somehost3")); assertFalse(hfp.getHosts().contains("somehost5")); assertFalse(hfp.getHosts().contains("somehost4")); assertTrue(hfp.getExcludedHosts().contains("somehost3")); assertFalse(hfp.getExcludedHosts().contains("somehost5")); assertFalse(hfp.getExcludedHosts().contains("somehost4")); }