public static String getApplicationWebURLOnJHSWithoutScheme(Configuration conf, ApplicationId appId) throws UnknownHostException { //construct the history url for job String addr = getJHSWebappURLWithoutScheme(conf); Iterator<String> it = ADDR_SPLITTER.split(addr).iterator(); it.next(); // ignore the bind host String port = it.next(); // Use hs address to figure out the host for webapp addr = conf.get(XLearningConfiguration.XLEARNING_HISTORY_ADDRESS, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_ADDRESS); String host = ADDR_SPLITTER.split(addr).iterator().next(); String hsAddress = JOINER.join(host, ":", port); InetSocketAddress address = NetUtils.createSocketAddr( hsAddress, getDefaultJHSWebappPort(), getDefaultJHSWebappURLWithoutScheme()); StringBuffer sb = new StringBuffer(); if (address.getAddress().isAnyLocalAddress() || address.getAddress().isLoopbackAddress()) { sb.append(InetAddress.getLocalHost().getCanonicalHostName()); } else { sb.append(address.getHostName()); } sb.append(":").append(address.getPort()); sb.append("/jobhistory/job/"); JobID jobId = TypeConverter.fromYarn(appId); sb.append(jobId.toString()); return sb.toString(); }
public FixedServiceInstance(String host) { if (resolveHosts) { try { InetAddress inetAddress = InetAddress.getByName(host); if (NetUtils.isLocalAddress(inetAddress)) { InetSocketAddress socketAddress = new InetSocketAddress(0); socketAddress = NetUtils.getConnectAddress(socketAddress); LOG.info("Adding host identified as local: " + host + " as " + socketAddress.getHostName()); host = socketAddress.getHostName(); } } catch (UnknownHostException e) { LOG.warn("Ignoring resolution issues for host: " + host, e); } } this.host = host; final URL serviceURL; try { serviceURL = new URL(LlapFixedRegistryImpl.this.webScheme, host, LlapFixedRegistryImpl.this.webPort, ""); this.serviceAddress = serviceURL.toString(); } catch (MalformedURLException e) { throw new RuntimeException(e); } }
/** * Returns an InetSocketAddress that a client can use to connect to the * given listening address. * * @param addr of a listener * @return socket address that a client can use to connect to the server. */ public static InetSocketAddress getConnectAddress(InetSocketAddress addr) { if (!addr.isUnresolved() && addr.getAddress().isAnyLocalAddress()) { try { addr = new InetSocketAddress(InetAddress.getLocalHost(), addr.getPort()); } catch (UnknownHostException uhe) { // shouldn't get here unless the host doesn't have a loopback iface addr = createSocketAddrForHost("127.0.0.1", addr.getPort()); } } return addr; }
public LlapManagementProtocolClientImpl(Configuration conf, String hostname, int port, @Nullable RetryPolicy retryPolicy, @Nullable SocketFactory socketFactory) { this.conf = conf; this.serverAddr = NetUtils.createSocketAddr(hostname, port); this.retryPolicy = retryPolicy; if (socketFactory == null) { this.socketFactory = NetUtils.getDefaultSocketFactory(conf); } else { this.socketFactory = socketFactory; } }
protected void checkHdfsUriForTimeout(Configuration config) throws IOException { URI hdfsUri = FileSystem.getDefaultUri(config); String address = hdfsUri.getAuthority(); int port = hdfsUri.getPort(); if (address == null || address.isEmpty() || port < 0) { return; } InetSocketAddress namenode = NetUtils.createSocketAddr(address, port); SocketFactory socketFactory = NetUtils.getDefaultSocketFactory(config); Socket socket = null; try { socket = socketFactory.createSocket(); NetUtils.connect(socket, namenode, 1000); // 1 second timeout } finally { IOUtils.closeQuietly(socket); } }
@Test public void testGetConnectAddress() throws IOException { NetUtils.addStaticResolution("host", "127.0.0.1"); InetSocketAddress addr = NetUtils.createSocketAddrForHost("host", 1); InetSocketAddress connectAddr = NetUtils.getConnectAddress(addr); assertEquals(addr.getHostName(), connectAddr.getHostName()); addr = new InetSocketAddress(1); connectAddr = NetUtils.getConnectAddress(addr); assertEquals(InetAddress.getLocalHost().getHostName(), connectAddr.getHostName()); }
public void testUpdateSocketAddress() throws IOException { InetSocketAddress addr = NetUtils.createSocketAddrForHost("host", 1); InetSocketAddress connectAddr = conf.updateConnectAddr("myAddress", addr); assertEquals(connectAddr.getHostName(), addr.getHostName()); addr = new InetSocketAddress(1); connectAddr = conf.updateConnectAddr("myAddress", addr); assertEquals(connectAddr.getHostName(), InetAddress.getLocalHost().getHostName()); }
public static RPC.Server startProtocolServer(int srvPort, int numHandlers, AtomicReference<InetSocketAddress> bindAddress, Configuration conf, BlockingService impl, Class<?> protocolClass, SecretManager<?> secretManager, PolicyProvider provider, ConfVars... aclVars) { InetSocketAddress addr = new InetSocketAddress(srvPort); RPC.Server server; try { server = createRpcServer(protocolClass, addr, conf, numHandlers, impl, secretManager, provider, aclVars); server.start(); } catch (IOException e) { LOG.error("Failed to run RPC Server on port: " + srvPort, e); throw new RuntimeException(e); } InetSocketAddress serverBindAddress = NetUtils.getConnectAddress(server); InetSocketAddress bindAddressVal = NetUtils.createSocketAddrForHost( serverBindAddress.getAddress().getCanonicalHostName(), serverBindAddress.getPort()); if (bindAddress != null) { bindAddress.set(bindAddressVal); } LOG.info("Instantiated " + protocolClass.getSimpleName() + " at " + bindAddressVal); return server; }
this.blockManager = blockManager; this.useDfsNetworkTopology = conf.getBoolean( DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_KEY, DFSConfigKeys.DFS_USE_DFS_NETWORK_TOPOLOGY_DEFAULT); new SlowDiskTracker(conf, timer) : null; this.defaultXferPort = NetUtils.createSocketAddr( conf.getTrimmed(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, DFSConfigKeys.DFS_DATANODE_ADDRESS_DEFAULT)).getPort(); this.defaultInfoPort = NetUtils.createSocketAddr( conf.getTrimmed(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY, DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_DEFAULT)).getPort(); this.defaultInfoSecurePort = NetUtils.createSocketAddr( conf.getTrimmed(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_DEFAULT)).getPort(); this.defaultIpcPort = NetUtils.createSocketAddr( conf.getTrimmed(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_DEFAULT)).getPort(); final ArrayList<String> locations = new ArrayList<>(); for (InetSocketAddress addr : hostConfigManager.getIncludes()) { locations.add(addr.getAddress().getHostAddress());
private InetSocketAddress startAndGetRPCServerAddress(InetSocketAddress serverAddress) { Configuration conf = new Configuration(); try { RPC.setProtocolEngine(conf, HAServiceProtocolPB.class, ProtobufRpcEngine.class); HAServiceProtocolServerSideTranslatorPB haServiceProtocolXlator = new HAServiceProtocolServerSideTranslatorPB(new MockHAProtocolImpl()); BlockingService haPbService = HAServiceProtocolService .newReflectiveBlockingService(haServiceProtocolXlator); Server server = new RPC.Builder(conf) .setProtocol(HAServiceProtocolPB.class) .setInstance(haPbService) .setBindAddress(serverAddress.getHostName()) .setPort(serverAddress.getPort()).build(); server.start(); return NetUtils.getConnectAddress(server); } catch (IOException e) { return null; } }
@BeforeClass public static void start() { InetSocketAddress address = new InetSocketAddress(0); Configuration configuration = new Configuration(); ResourceTracker instance = new ResourceTrackerTestImpl(); server = RpcServerFactoryPBImpl.get().getServer(ResourceTracker.class, instance, address, configuration, null, 1); server.start(); client = (ResourceTracker) RpcClientFactoryPBImpl.get().getClient( ResourceTracker.class, 1, NetUtils.getConnectAddress(server), configuration); }
/** * Get the socket address for <code>hostProperty</code> as a * <code>InetSocketAddress</code>. If <code>hostProperty</code> is * <code>null</code>, <code>addressProperty</code> will be used. This * is useful for cases where we want to differentiate between host * bind address and address clients should use to establish connection. * * @param hostProperty bind host property name. * @param addressProperty address property name. * @param defaultAddressValue the default value * @param defaultPort the default port * @return InetSocketAddress */ public InetSocketAddress getSocketAddr( String hostProperty, String addressProperty, String defaultAddressValue, int defaultPort) { InetSocketAddress bindAddr = getSocketAddr( addressProperty, defaultAddressValue, defaultPort); final String host = get(hostProperty); if (host == null || host.isEmpty()) { return bindAddr; } return NetUtils.createSocketAddr( host, bindAddr.getPort(), hostProperty); }
/** * Set the socket address a client can use to connect for the * <code>name</code> property as a <code>host:port</code>. The wildcard * address is replaced with the local host's address. If the host and address * properties are configured the host component of the address will be combined * with the port component of the addr to generate the address. This is to allow * optional control over which host name is used in multi-home bind-host * cases where a host can have multiple names * @param hostProperty the bind-host configuration name * @param addressProperty the service address configuration name * @param defaultAddressValue the service default address configuration value * @param addr InetSocketAddress of the service listener * @return InetSocketAddress for clients to connect */ public InetSocketAddress updateConnectAddr( String hostProperty, String addressProperty, String defaultAddressValue, InetSocketAddress addr) { final String host = get(hostProperty); final String connectHostPort = getTrimmed(addressProperty, defaultAddressValue); if (host == null || host.isEmpty() || connectHostPort == null || connectHostPort.isEmpty()) { //not our case, fall back to original logic return updateConnectAddr(addressProperty, addr); } final String connectHost = connectHostPort.split(":")[0]; // Create connect address using client address hostname and server port. return updateConnectAddr(addressProperty, NetUtils.createSocketAddrForHost( connectHost, addr.getPort())); }
@Test public void testXFrameHeaderSameOrigin() throws Exception { Configuration conf = new Configuration(); conf.set("hbase.http.filter.xframeoptions.mode", "SAMEORIGIN"); HttpServer myServer = new HttpServer.Builder().setName("test") .addEndpoint(new URI("http://localhost:0")) .setFindPort(true).setConf(conf).build(); myServer.setAttribute(HttpServer.CONF_CONTEXT_ATTRIBUTE, conf); myServer.addServlet("echo", "/echo", EchoServlet.class); myServer.start(); String serverURL = "http://" + NetUtils.getHostPortString(myServer.getConnectorAddress(0)); URL url = new URL(new URL(serverURL), "/echo?a=b&c=d"); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode()); assertEquals("SAMEORIGIN", conn.getHeaderField("X-Frame-Options")); myServer.stop(); }
private static void secureLogin(Configuration conf) throws IOException { UserGroupInformation.setConfiguration(conf); String addr = conf.get(DFSConfigKeys.DFS_SPS_ADDRESS_KEY, DFSConfigKeys.DFS_SPS_ADDRESS_DEFAULT); InetSocketAddress socAddr = NetUtils.createSocketAddr(addr, 0, DFSConfigKeys.DFS_SPS_ADDRESS_KEY); SecurityUtil.login(conf, DFSConfigKeys.DFS_SPS_KEYTAB_FILE_KEY, DFSConfigKeys.DFS_SPS_KERBEROS_PRINCIPAL_KEY, socAddr.getHostName()); }
if (ticket != null && ticket.hasKerberosCredentials()) { KerberosInfo krbInfo = remoteId.getProtocol().getAnnotation(KerberosInfo.class); if (krbInfo != null) { String principal = ticket.getUserName(); String host = SecurityUtil.getHostFromPrincipal(principal); InetAddress localAddr = NetUtils.getLocalInetAddress(host); if (localAddr != null) { this.socket.setReuseAddress(true); localAddr = NetUtils.bindToLocalAddress(localAddr, bindToWildCardAddress); LOG.debug("Binding {} to {}", principal, (bindToWildCardAddress) ? "0.0.0.0" : localAddr); this.socket.bind(new InetSocketAddress(localAddr, 0)); NetUtils.connect(this.socket, server, bindAddr, connectionTimeout); this.socket.setSoTimeout(soTimeout); return;
private void initIpcServer(Configuration conf) throws IOException { //init ipc server InetSocketAddress ipcAddr = NetUtils.createSocketAddr( conf.get("dfs.datanode.ipc.address")); ipcServer = RPC.getServer(this, ipcAddr.getHostName(), ipcAddr.getPort(), conf.getInt("dfs.datanode.handler.count", 3), false, conf); ipcServer.start(); }
try { if (UserGroupInformation.isSecurityEnabled()) { final String daemonPrincipal = HiveConf.getVar(daemonConf, ConfVars.LLAP_KERBEROS_PRINCIPAL); final String daemonKeytab = HiveConf.getVar(daemonConf, ConfVars.LLAP_KERBEROS_KEYTAB_FILE); LlapUtil.loginWithKerberosAndUpdateCurrentUser(daemonPrincipal, daemonKeytab); String currentUser = UserGroupInformation.getCurrentUser().getShortUserName(); LOG.info("Starting daemon as user: {}", currentUser); daemonId = new DaemonId(currentUser, LlapUtil.generateClusterName(daemonConf), ", configured(exec + io if enabled)=" + LlapUtil.humanReadableByteCount(memRequired)); this.shuffleHandlerConf = new Configuration(daemonConf); this.shuffleHandlerConf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, shufflePort); this.shuffleHandlerConf.set(ShuffleHandler.SHUFFLE_HANDLER_LOCAL_DIRS, StringUtils.arrayToString(localDirs)); this.shuffleHandlerConf.setBoolean(ShuffleHandler.SHUFFLE_DIR_WATCHER_ENABLED, this.socketFactory = NetUtils.getDefaultSocketFactory(daemonConf); this.amReporter = new AMReporter(numExecutors, maxAmReporterThreads, srvAddress, new QueryFailedHandlerProxy(), daemonConf, daemonId, socketFactory);
@Test public void testServerAddress() throws IOException { Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class) .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0) .setNumHandlers(5).setVerbose(true).build(); InetSocketAddress bindAddr = null; try { bindAddr = NetUtils.getConnectAddress(server); } finally { server.stop(); } assertEquals(InetAddress.getLocalHost(), bindAddr.getAddress()); }
/** * Update the server address if the address corresponding to the host * name has changed. * * @return true if an addr change was detected. * @throws IOException when the hostname cannot be resolved. */ private synchronized boolean updateAddress() throws IOException { // Do a fresh lookup with the old host name. InetSocketAddress currentAddr = NetUtils.createSocketAddrForHost( server.getHostName(), server.getPort()); if (!server.equals(currentAddr)) { LOG.warn("Address change detected. Old: " + server.toString() + " New: " + currentAddr.toString()); server = currentAddr; return true; } return false; }