/** * Lookup the HTTP / HTTPS address of the namenode, and replace its hostname * with defaultHost when it found out that the address is a wildcard / local * address. * * @param defaultHost * The default host name of the namenode. * @param conf * The configuration * @param scheme * HTTP or HTTPS * @throws IOException */ public static URI getInfoServerWithDefaultHost(String defaultHost, Configuration conf, final String scheme) throws IOException { URI configuredAddr = getInfoServer(null, conf, scheme); String authority = substituteForWildcardAddress( configuredAddr.getAuthority(), defaultHost); return URI.create(scheme + "://" + authority); }
/** * Derive the namenode http address from the current file system, * either default or as set by "-fs" in the generic options. * @return Returns http address or null if failure. * @throws IOException if we can't determine the active NN address */ private URI getCurrentNamenodeAddress(Path target) throws IOException { //String nnAddress = null; Configuration conf = getConf(); //get the filesystem object to verify it is an HDFS system final FileSystem fs = target.getFileSystem(conf); if (!(fs instanceof DistributedFileSystem)) { System.err.println("FileSystem is " + fs.getUri()); return null; } return DFSUtil.getInfoServer(HAUtil.getAddressOfActive(fs), conf, DFSUtil.getHttpClientScheme(conf)); }
String stacksUrl = DFSUtil.getInfoServer(localNNTarget.getAddress(), conf, DFSUtil.getHttpClientScheme(conf)) + "/stacks"; URL url = new URL(stacksUrl);
private NamespaceInfo handshake(Configuration conf) throws IOException { // connect to name node InetSocketAddress nnAddress = NameNode.getServiceAddress(conf, true); this.namenode = NameNodeProxies.createNonHAProxy(conf, nnAddress, NamenodeProtocol.class, UserGroupInformation.getCurrentUser(), true).getProxy(); this.nnRpcAddress = NetUtils.getHostPortString(nnAddress); this.nnHttpAddress = DFSUtil.getInfoServer(nnAddress, conf, DFSUtil.getHttpClientScheme(conf)).toURL(); // get version and id info from the name-node NamespaceInfo nsInfo = null; while(!isStopRequested()) { try { nsInfo = handshake(namenode); break; } catch(SocketTimeoutException e) { // name-node is busy LOG.info("Problem connecting to server: " + nnAddress); try { Thread.sleep(1000); } catch (InterruptedException ie) { LOG.warn("Encountered exception ", e); } } } return nsInfo; }
/** * Download the most recent fsimage from the name node, and save it to a local * file in the given directory. * * @param argv * List of of command line parameters. * @param idx * The index of the command that is being processed. * @return an exit code indicating success or failure. * @throws IOException */ public int fetchImage(final String[] argv, final int idx) throws IOException { Configuration conf = getConf(); final URL infoServer = DFSUtil.getInfoServer( HAUtil.getAddressOfActive(getDFS()), conf, DFSUtil.getHttpClientScheme(conf)).toURL(); SecurityUtil.doAsCurrentUser(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { TransferFsImage.downloadMostRecentImageToDirectory(infoServer, new File(argv[idx])); return null; } }); return 0; }
NamenodeMXBeanHelper(InetSocketAddress addr, Configuration conf) throws IOException, MalformedObjectNameException { this.host = addr.getHostName(); this.httpAddress = DFSUtil.getInfoServer(addr, conf, DFSUtil.getHttpClientScheme(conf)); }
NamenodeMXBeanHelper(InetSocketAddress addr, Configuration conf) throws IOException, MalformedObjectNameException { this.host = addr.getHostName(); this.httpAddress = DFSUtil.getInfoServer(addr, conf, DFSUtil.getHttpClientScheme(conf)); }
/** * Lookup the HTTP / HTTPS address of the namenode, and replace its hostname * with defaultHost when it found out that the address is a wildcard / local * address. * * @param defaultHost * The default host name of the namenode. * @param conf * The configuration * @param scheme * HTTP or HTTPS * @throws IOException */ public static URI getInfoServerWithDefaultHost(String defaultHost, Configuration conf, final String scheme) throws IOException { URI configuredAddr = getInfoServer(null, conf, scheme); String authority = substituteForWildcardAddress( configuredAddr.getAuthority(), defaultHost); return URI.create(scheme + "://" + authority); }
/** * Lookup the HTTP / HTTPS address of the namenode, and replace its hostname * with defaultHost when it found out that the address is a wildcard / local * address. * * @param defaultHost * The default host name of the namenode. * @param conf * The configuration * @param scheme * HTTP or HTTPS * @throws IOException */ public static URI getInfoServerWithDefaultHost(String defaultHost, Configuration conf, final String scheme) throws IOException { URI configuredAddr = getInfoServer(null, conf, scheme); String authority = substituteForWildcardAddress( configuredAddr.getAuthority(), defaultHost); return URI.create(scheme + "://" + authority); }
/** * Derive the namenode http address from the current file system, * either default or as set by "-fs" in the generic options. * @return Returns http address or null if failure. * @throws IOException if we can't determine the active NN address */ private URI getCurrentNamenodeAddress(Path target) throws IOException { //String nnAddress = null; Configuration conf = getConf(); //get the filesystem object to verify it is an HDFS system final FileSystem fs = target.getFileSystem(conf); if (!(fs instanceof DistributedFileSystem)) { System.err.println("FileSystem is " + fs.getUri()); return null; } return DFSUtil.getInfoServer(HAUtil.getAddressOfActive(fs), conf, DFSUtil.getHttpClientScheme(conf)); }
/** * Derive the namenode http address from the current file system, * either default or as set by "-fs" in the generic options. * @return Returns http address or null if failure. * @throws IOException if we can't determine the active NN address */ private URI getCurrentNamenodeAddress(Path target) throws IOException { //String nnAddress = null; Configuration conf = getConf(); //get the filesystem object to verify it is an HDFS system final FileSystem fs = target.getFileSystem(conf); if (!(fs instanceof DistributedFileSystem)) { System.err.println("FileSystem is " + fs.getUri()); return null; } return DFSUtil.getInfoServer(HAUtil.getAddressOfActive(fs), conf, DFSUtil.getHttpClientScheme(conf)); }
private NamespaceInfo handshake(Configuration conf) throws IOException { // connect to name node InetSocketAddress nnAddress = NameNode.getServiceAddress(conf, true); this.namenode = NameNodeProxies.createNonHAProxy(conf, nnAddress, NamenodeProtocol.class, UserGroupInformation.getCurrentUser(), true).getProxy(); this.nnRpcAddress = NetUtils.getHostPortString(nnAddress); this.nnHttpAddress = DFSUtil.getInfoServer(nnAddress, conf, DFSUtil.getHttpClientScheme(conf)).toURL(); // get version and id info from the name-node NamespaceInfo nsInfo = null; while(!isStopRequested()) { try { nsInfo = handshake(namenode); break; } catch(SocketTimeoutException e) { // name-node is busy LOG.info("Problem connecting to server: " + nnAddress); try { Thread.sleep(1000); } catch (InterruptedException ie) { LOG.warn("Encountered exception ", e); } } } return nsInfo; }
@Test public void testGetInfoServer() throws IOException, URISyntaxException { HdfsConfiguration conf = new HdfsConfiguration(); URI httpsport = DFSUtil.getInfoServer(null, conf, "https"); assertEquals(new URI("https", null, "0.0.0.0", DFS_NAMENODE_HTTPS_PORT_DEFAULT, null, null, null), httpsport); URI httpport = DFSUtil.getInfoServer(null, conf, "http"); assertEquals(new URI("http", null, "0.0.0.0", DFS_NAMENODE_HTTP_PORT_DEFAULT, null, null, null), httpport); URI httpAddress = DFSUtil.getInfoServer(new InetSocketAddress( "localhost", 8020), conf, "http"); assertEquals( URI.create("http://localhost:" + DFS_NAMENODE_HTTP_PORT_DEFAULT), httpAddress); }
/** * Download the most recent fsimage from the name node, and save it to a local * file in the given directory. * * @param argv * List of of command line parameters. * @param idx * The index of the command that is being processed. * @return an exit code indicating success or failure. * @throws IOException */ public int fetchImage(final String[] argv, final int idx) throws IOException { Configuration conf = getConf(); final URL infoServer = DFSUtil.getInfoServer( HAUtil.getAddressOfActive(getDFS()), conf, DFSUtil.getHttpClientScheme(conf)).toURL(); SecurityUtil.doAsCurrentUser(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { TransferFsImage.downloadMostRecentImageToDirectory(infoServer, new File(argv[idx])); return null; } }); return 0; }
/** * Download the most recent fsimage from the name node, and save it to a local * file in the given directory. * * @param argv * List of of command line parameters. * @param idx * The index of the command that is being processed. * @return an exit code indicating success or failure. * @throws IOException */ public int fetchImage(final String[] argv, final int idx) throws IOException { Configuration conf = getConf(); final URL infoServer = DFSUtil.getInfoServer( HAUtil.getAddressOfActive(getDFS()), conf, DFSUtil.getHttpClientScheme(conf)).toURL(); SecurityUtil.doAsCurrentUser(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { TransferFsImage.downloadMostRecentImageToDirectory(infoServer, new File(argv[idx])); return null; } }); return 0; }
private NamespaceInfo handshake(Configuration conf) throws IOException { // connect to name node InetSocketAddress nnAddress = NameNode.getServiceAddress(conf, true); this.namenode = NameNodeProxies.createNonHAProxy(conf, nnAddress, NamenodeProtocol.class, UserGroupInformation.getCurrentUser(), true).getProxy(); this.nnRpcAddress = NetUtils.getHostPortString(nnAddress); this.nnHttpAddress = DFSUtil.getInfoServer(nnAddress, conf, DFSUtil.getHttpClientScheme(conf)).toURL(); // get version and id info from the name-node NamespaceInfo nsInfo = null; while(!isStopRequested()) { try { nsInfo = handshake(namenode); break; } catch(SocketTimeoutException e) { // name-node is busy LOG.info("Problem connecting to server: " + nnAddress); try { Thread.sleep(1000); } catch (InterruptedException ie) { LOG.warn("Encountered exception ", e); } } } return nsInfo; }
/** * Similar to the above test, except that there are multiple local files * and one of them can be saved. */ @Test public void testClientSideExceptionOnJustOneDir() throws IOException { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) .numDataNodes(0).build(); NNStorage mockStorage = Mockito.mock(NNStorage.class); List<File> localPaths = ImmutableList.of( new File("/xxxxx-does-not-exist/blah"), new File(TEST_DIR, "testfile") ); try { URL fsName = DFSUtil.getInfoServer( cluster.getNameNode().getServiceRpcAddress(), conf, DFSUtil.getHttpClientScheme(conf)).toURL(); String id = "getimage=1&txid=0"; TransferFsImage.getFileClient(fsName, id, localPaths, mockStorage, false); Mockito.verify(mockStorage).reportErrorOnFile(localPaths.get(0)); assertTrue("The valid local file should get saved properly", localPaths.get(1).length() > 0); } finally { cluster.shutdown(); } }
URL fsName = DFSUtil.getInfoServer( cluster.getNameNode().getServiceRpcAddress(), conf, DFSUtil.getHttpClientScheme(conf)).toURL();
URL fsName = DFSUtil.getInfoServer( cluster.getNameNode().getServiceRpcAddress(), conf, DFSUtil.getHttpClientScheme(conf)).toURL();
public NamenodeStatus getNamenodeStatus() throws IOException, MalformedObjectNameException { NamenodeStatus nn = new NamenodeStatus(); nn.address = this.address; nn.filesAndDirectories = mxbeanProxy.getTotalFiles(); nn.capacity = mxbeanProxy.getTotal(); nn.free = mxbeanProxy.getFree(); nn.nsUsed = mxbeanProxy.getNamespaceUsed(); nn.nonDfsUsed = mxbeanProxy.getNonDfsUsedSpace(); nn.blocksCount = mxbeanProxy.getTotalBlocks(); nn.missingBlocksCount = mxbeanProxy.getNumberOfMissingBlocks(); nn.httpAddress = DFSUtil.getInfoServer(rpcAddress, conf); nn.safeModeText = mxbeanProxy.getSafeModeText(); getLiveNodeCount(mxbeanProxy.getLiveNodes(), nn); getDeadNodeCount(mxbeanProxy.getDeadNodes(), nn); nn.namenodeSpecificInfo = mxbeanProxy.getNNSpecificKeys(); nn.isPrimary = mxbeanProxy.getIsPrimary(); return nn; }