/** * Resolve an HDFS URL into real INetSocketAddress. It works like a DNS * resolver when the URL points to an non-HA cluster. When the URL points to * an HA cluster with its logical name, the resolver further resolves the * logical name(i.e., the authority in the URL) into real namenode addresses. */ private InetSocketAddress[] resolveNNAddr() throws IOException { Configuration conf = getConf(); final String scheme = uri.getScheme(); ArrayList<InetSocketAddress> ret = new ArrayList<InetSocketAddress>(); if (!HAUtil.isLogicalUri(conf, uri)) { InetSocketAddress addr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort()); ret.add(addr); } else { Map<String, Map<String, InetSocketAddress>> addresses = DFSUtil .getHaNnWebHdfsAddresses(conf, scheme); // Extract the entry corresponding to the logical name. Map<String, InetSocketAddress> addrs = addresses.get(uri.getHost()); for (InetSocketAddress addr : addrs.values()) { ret.add(addr); } } InetSocketAddress[] r = new InetSocketAddress[ret.size()]; return ret.toArray(r); }
/** * Resolve an HDFS URL into real INetSocketAddress. It works like a DNS * resolver when the URL points to an non-HA cluster. When the URL points to * an HA cluster with its logical name, the resolver further resolves the * logical name(i.e., the authority in the URL) into real namenode addresses. */ private InetSocketAddress[] resolveNNAddr() throws IOException { Configuration conf = getConf(); final String scheme = uri.getScheme(); ArrayList<InetSocketAddress> ret = new ArrayList<InetSocketAddress>(); if (!HAUtil.isLogicalUri(conf, uri)) { InetSocketAddress addr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort()); ret.add(addr); } else { Map<String, Map<String, InetSocketAddress>> addresses = DFSUtil .getHaNnWebHdfsAddresses(conf, scheme); // Extract the entry corresponding to the logical name. Map<String, InetSocketAddress> addrs = addresses.get(uri.getHost()); for (InetSocketAddress addr : addrs.values()) { ret.add(addr); } } InetSocketAddress[] r = new InetSocketAddress[ret.size()]; return ret.toArray(r); }
@Override protected URI canonicalizeUri(URI uri) { if (HAUtil.isLogicalUri(getConf(), uri)) { // Don't try to DNS-resolve logical URIs, since the 'authority' // portion isn't a proper hostname return uri; } else { return NetUtils.getCanonicalUri(uri, getDefaultPort()); } }
@Override protected URI canonicalizeUri(URI uri) { if (HAUtil.isLogicalUri(getConf(), uri)) { // Don't try to DNS-resolve logical URIs, since the 'authority' // portion isn't a proper hostname return uri; } else { return NetUtils.getCanonicalUri(uri, getDefaultPort()); } }
Configuration dfsConf = dfs.getConf(); URI dfsUri = dfs.getUri(); boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri);
private static ClientProtocol getNNProxy( Token<DelegationTokenIdentifier> token, Configuration conf) throws IOException { URI uri = HAUtil.getServiceUriFromToken(HdfsConstants.HDFS_URI_SCHEME, token); if (HAUtil.isTokenForLogicalUri(token) && !HAUtil.isLogicalUri(conf, uri)) { // If the token is for a logical nameservice, but the configuration // we have disagrees about that, we can't actually renew it. // This can be the case in MR, for example, if the RM doesn't // have all of the HA clusters configured in its configuration. throw new IOException("Unable to map logical nameservice URI '" + uri + "' to a NameNode. Local configuration does not have " + "a failover proxy provider configured."); } NameNodeProxies.ProxyAndInfo<ClientProtocol> info = NameNodeProxies.createProxy(conf, uri, ClientProtocol.class); assert info.getDelegationTokenService().equals(token.getService()) : "Returned service '" + info.getDelegationTokenService().toString() + "' doesn't match expected service '" + token.getService().toString() + "'"; return info.getProxy(); }
private static ClientProtocol getNNProxy( Token<DelegationTokenIdentifier> token, Configuration conf) throws IOException { URI uri = HAUtil.getServiceUriFromToken(HdfsConstants.HDFS_URI_SCHEME, token); if (HAUtil.isTokenForLogicalUri(token) && !HAUtil.isLogicalUri(conf, uri)) { // If the token is for a logical nameservice, but the configuration // we have disagrees about that, we can't actually renew it. // This can be the case in MR, for example, if the RM doesn't // have all of the HA clusters configured in its configuration. throw new IOException("Unable to map logical nameservice URI '" + uri + "' to a NameNode. Local configuration does not have " + "a failover proxy provider configured."); } NameNodeProxies.ProxyAndInfo<ClientProtocol> info = NameNodeProxies.createProxy(conf, uri, ClientProtocol.class); assert info.getDelegationTokenService().equals(token.getService()) : "Returned service '" + info.getDelegationTokenService().toString() + "' doesn't match expected service '" + token.getService().toString() + "'"; return info.getProxy(); }
/** * Command to ask the namenode to reread the hosts and excluded hosts * file. * Usage: hdfs dfsadmin -refreshNodes * @exception IOException */ public int refreshNodes() throws IOException { int exitCode = -1; DistributedFileSystem dfs = getDFS(); Configuration dfsConf = dfs.getConf(); URI dfsUri = dfs.getUri(); boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri); if (isHaEnabled) { String nsId = dfsUri.getHost(); List<ProxyAndInfo<ClientProtocol>> proxies = HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf, nsId, ClientProtocol.class); for (ProxyAndInfo<ClientProtocol> proxy: proxies) { proxy.getProxy().refreshNodes(); System.out.println("Refresh nodes successful for " + proxy.getAddress()); } } else { dfs.refreshNodes(); System.out.println("Refresh nodes successful"); } exitCode = 0; return exitCode; }
/** * Command to ask the namenode to save the namespace. * Usage: hdfs dfsadmin -saveNamespace * @exception IOException * @see org.apache.hadoop.hdfs.protocol.ClientProtocol#saveNamespace() */ public int saveNamespace() throws IOException { int exitCode = -1; DistributedFileSystem dfs = getDFS(); Configuration dfsConf = dfs.getConf(); URI dfsUri = dfs.getUri(); boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri); if (isHaEnabled) { String nsId = dfsUri.getHost(); List<ProxyAndInfo<ClientProtocol>> proxies = HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf, nsId, ClientProtocol.class); for (ProxyAndInfo<ClientProtocol> proxy : proxies) { proxy.getProxy().saveNamespace(); System.out.println("Save namespace successful for " + proxy.getAddress()); } } else { dfs.saveNamespace(); System.out.println("Save namespace successful"); } exitCode = 0; return exitCode; }
/** * Command to ask the namenode to save the namespace. * Usage: hdfs dfsadmin -saveNamespace * @exception IOException * @see org.apache.hadoop.hdfs.protocol.ClientProtocol#saveNamespace() */ public int saveNamespace() throws IOException { int exitCode = -1; DistributedFileSystem dfs = getDFS(); Configuration dfsConf = dfs.getConf(); URI dfsUri = dfs.getUri(); boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri); if (isHaEnabled) { String nsId = dfsUri.getHost(); List<ProxyAndInfo<ClientProtocol>> proxies = HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf, nsId, ClientProtocol.class); for (ProxyAndInfo<ClientProtocol> proxy : proxies) { proxy.getProxy().saveNamespace(); System.out.println("Save namespace successful for " + proxy.getAddress()); } } else { dfs.saveNamespace(); System.out.println("Save namespace successful"); } exitCode = 0; return exitCode; }
/** * Command to ask the namenode to reread the hosts and excluded hosts * file. * Usage: hdfs dfsadmin -refreshNodes * @exception IOException */ public int refreshNodes() throws IOException { int exitCode = -1; DistributedFileSystem dfs = getDFS(); Configuration dfsConf = dfs.getConf(); URI dfsUri = dfs.getUri(); boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri); if (isHaEnabled) { String nsId = dfsUri.getHost(); List<ProxyAndInfo<ClientProtocol>> proxies = HAUtil.getProxiesForAllNameNodesInNameservice(dfsConf, nsId, ClientProtocol.class); for (ProxyAndInfo<ClientProtocol> proxy: proxies) { proxy.getProxy().refreshNodes(); System.out.println("Refresh nodes successful for " + proxy.getAddress()); } } else { dfs.refreshNodes(); System.out.println("Refresh nodes successful"); } exitCode = 0; return exitCode; }
boolean isHaEnabled = HAUtil.isLogicalUri(conf, dfsUri);
boolean isHaEnabled = HAUtil.isLogicalUri(conf, dfsUri);
Token<DelegationTokenIdentifier> delegationToken() throws IOException { String delegation = param(DelegationParam.NAME); final Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(); token.decodeFromUrlString(delegation); URI nnUri = URI.create(HDFS_URI_SCHEME + "://" + namenodeId()); boolean isLogical = HAUtil.isLogicalUri(conf, nnUri); if (isLogical) { token.setService(HAUtil.buildTokenServiceForLogicalUri(nnUri, HDFS_URI_SCHEME)); } else { token.setService(SecurityUtil.buildTokenService(nnUri)); } return token; }
boolean isHaEnabled = HAUtil.isLogicalUri(conf, dfsUri);
Configuration dfsConf = dfs.getConf(); URI dfsUri = dfs.getUri(); boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri);
Token<DelegationTokenIdentifier> delegationToken() throws IOException { String delegation = param(DelegationParam.NAME); final Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(); token.decodeFromUrlString(delegation); URI nnUri = URI.create(HDFS_URI_SCHEME + "://" + namenodeId()); boolean isLogical = HAUtil.isLogicalUri(conf, nnUri); if (isLogical) { token.setService(HAUtil.buildTokenServiceForLogicalUri(nnUri, HDFS_URI_SCHEME)); } else { token.setService(SecurityUtil.buildTokenService(nnUri)); } return token; }
boolean isHaAndLogicalUri = HAUtil.isLogicalUri(dfsConf, dfsUri); if (isHaAndLogicalUri) {
boolean isHaAndLogicalUri = HAUtil.isLogicalUri(dfsConf, dfsUri); if (isHaAndLogicalUri) {
Configuration dfsConf = dfs.getConf(); URI dfsUri = dfs.getUri(); boolean isHaEnabled = HAUtil.isLogicalUri(dfsConf, dfsUri);