private static HostSet readFile(String type, String filename) throws IOException { HostSet res = new HostSet(); if (!filename.isEmpty()) { HashSet<String> entrySet = new HashSet<String>(); HostsFileReader.readFileToSet(type, filename, entrySet); for (String str : entrySet) { InetSocketAddress addr = parseEntry(type, filename, str); if (addr != null) { res.add(addr); } } } return res; }
/** * Read the includes and excludes lists from the named files. Any previous * includes and excludes lists are discarded. * @param includeFile the path to the new includes list * @param excludeFile the path to the new excludes list * @throws IOException thrown if there is a problem reading one of the files */ private void refresh(String includeFile, String excludeFile) throws IOException { HostSet newIncludes = readFile("included", includeFile); HostSet newExcludes = readFile("excluded", excludeFile); refresh(newIncludes, newExcludes); }
@Override public synchronized boolean isExcluded(DatanodeID dn) { return isExcluded(dn.getResolvedAddress()); }
final HostFileManager.HostSet includedNodes = hostFileManager.getIncludes(); final HostFileManager.HostSet excludedNodes = hostFileManager.getExcludes(); (listDeadNodes && isDead) || (listDecommissioningNodes && isDecommissioning)) && hostFileManager.isIncluded(dn)) { nodes.add(dn); foundNodes.add(HostFileManager.resolvedAddressFromDatanodeID(dn)); "includedNodes = " + hostFileManager.getIncludes() + ", excludedNodes = " + hostFileManager.getExcludes() + ", foundNodes = " + foundNodes + ", nodes = " + nodes);
DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_DEFAULT)).getPort(); try { this.hostFileManager.refresh(conf.get(DFSConfigKeys.DFS_HOSTS, ""), conf.get(DFSConfigKeys.DFS_HOSTS_EXCLUDE, "")); } catch (IOException e) { for (InetSocketAddress addr : hostFileManager.getIncludes()) { locations.add(addr.getAddress().getHostAddress());
/** * 1. Added to hosts --> no further work needed here. * 2. Removed from hosts --> mark AdminState as decommissioned. * 3. Added to exclude --> start decommission. * 4. Removed from exclude --> stop decommission. */ private void refreshDatanodes() { for(DatanodeDescriptor node : datanodeMap.values()) { // Check if not include. if (!hostFileManager.isIncluded(node)) { node.setDisallowed(true); // case 2. } else { if (hostFileManager.isExcluded(node)) { decomManager.startDecommission(node); // case 3. } else { decomManager.stopDecommission(node); // case 4. } } } }
HostFileManager hm = new HostFileManager(); HostFileManager.HostSet noNodes = new HostFileManager.HostSet(); HostFileManager.HostSet oneNode = new HostFileManager.HostSet(); oneNode.add(entry("127.0.0.1:23456")); hm.refresh(twoNodes, noNodes); Whitebox.setInternalState(dm, "hostFileManager", hm); hm.refresh(oneNode, noNodes); hm.refresh(noNodes, noNodes);
@Override public void refresh() throws IOException { refresh(conf.get(DFSConfigKeys.DFS_HOSTS, ""), conf.get(DFSConfigKeys.DFS_HOSTS_EXCLUDE, "")); } private static HostSet readFile(String type, String filename)
synchronized boolean isExcluded(DatanodeID dn) { return excludes.match(resolvedAddressFromDatanodeID(dn)); }
if (!hostFileManager.isIncluded(nodeReg)) { throw new DisallowedDatanodeException(nodeReg);
final HostFileManager.HostSet includedNodes = hostFileManager.getIncludes(); final HostFileManager.HostSet excludedNodes = hostFileManager.getExcludes(); (listDeadNodes && isDead) || (listDecommissioningNodes && isDecommissioning)) && hostFileManager.isIncluded(dn)) { nodes.add(dn); foundNodes.add(HostFileManager.resolvedAddressFromDatanodeID(dn)); "includedNodes = " + hostFileManager.getIncludes() + ", excludedNodes = " + hostFileManager.getExcludes() + ", foundNodes = " + foundNodes + ", nodes = " + nodes);
DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_DEFAULT)).getPort(); try { this.hostFileManager.refresh(conf.get(DFSConfigKeys.DFS_HOSTS, ""), conf.get(DFSConfigKeys.DFS_HOSTS_EXCLUDE, "")); } catch (IOException e) { for (InetSocketAddress addr : hostFileManager.getIncludes()) { locations.add(addr.getAddress().getHostAddress());
/** * 1. Added to hosts --> no further work needed here. * 2. Removed from hosts --> mark AdminState as decommissioned. * 3. Added to exclude --> start decommission. * 4. Removed from exclude --> stop decommission. */ private void refreshDatanodes() { for(DatanodeDescriptor node : datanodeMap.values()) { // Check if not include. if (!hostFileManager.isIncluded(node)) { node.setDisallowed(true); // case 2. } else { if (hostFileManager.isExcluded(node)) { decomManager.startDecommission(node); // case 3. } else { decomManager.stopDecommission(node); // case 4. } } } }
FSNamesystem fsn = mock(FSNamesystem.class); Configuration conf = new Configuration(); HostFileManager hm = new HostFileManager(); HostFileManager.HostSet includedNodes = new HostFileManager.HostSet(); HostFileManager.HostSet excludedNodes = new HostFileManager.HostSet(); Assert.assertEquals(2, excludedNodes.size()); hm.refresh(includedNodes, excludedNodes);
/** Reread include/exclude files. */ private void refreshHostsReader(Configuration conf) throws IOException { // Reread the conf to get dfs.hosts and dfs.hosts.exclude filenames. // Update the file names and refresh internal includes and excludes list. if (conf == null) { conf = new HdfsConfiguration(); } this.hostFileManager.refresh(conf.get(DFSConfigKeys.DFS_HOSTS, ""), conf.get(DFSConfigKeys.DFS_HOSTS_EXCLUDE, "")); }
synchronized boolean isExcluded(DatanodeID dn) { return excludes.match(resolvedAddressFromDatanodeID(dn)); }
if (!hostFileManager.isIncluded(nodeReg)) { throw new DisallowedDatanodeException(nodeReg);
/** * Read the includes and excludes lists from the named files. Any previous * includes and excludes lists are discarded. * @param includeFile the path to the new includes list * @param excludeFile the path to the new excludes list * @throws IOException thrown if there is a problem reading one of the files */ void refresh(String includeFile, String excludeFile) throws IOException { HostSet newIncludes = readFile("included", includeFile); HostSet newExcludes = readFile("excluded", excludeFile); refresh(newIncludes, newExcludes); }
/** * Create an InetSocketAddress for a host:port string * @param host a host identifier in host:port format * @return a corresponding InetSocketAddress object */ private static InetSocketAddress entry(String host) { return HostFileManager.parseEntry("dummy", "dummy", host); }
/** Reread include/exclude files. */ private void refreshHostsReader(Configuration conf) throws IOException { // Reread the conf to get dfs.hosts and dfs.hosts.exclude filenames. // Update the file names and refresh internal includes and excludes list. if (conf == null) { conf = new HdfsConfiguration(); } this.hostFileManager.refresh(conf.get(DFSConfigKeys.DFS_HOSTS, ""), conf.get(DFSConfigKeys.DFS_HOSTS_EXCLUDE, "")); }