private static void getJhToken(Configuration conf, Credentials cred) throws IOException { YarnRPC rpc = YarnRPC.create(conf); final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS); LOG.debug("Connecting to HistoryServer at: " + serviceAddr); HSClientProtocol hsProxy = (HSClientProtocol) rpc.getProxy(HSClientProtocol.class, NetUtils.createSocketAddr(serviceAddr), conf); LOG.info("Pre-fetching JH token from job history server"); Token<?> jhToken = null; try { jhToken = getDelegationTokenFromHS(hsProxy, conf); } catch (Exception exc) { throw new IOException("Failed to fetch JH token.", exc); } if (jhToken == null) { LOG.error("getDelegationTokenFromHS() returned null"); throw new IOException("Unable to fetch JH token."); } LOG.info("Created JH token: " + jhToken.toString()); LOG.info("Token kind: " + jhToken.getKind()); LOG.info("Token id: " + Arrays.toString(jhToken.getIdentifier())); LOG.info("Token service: " + jhToken.getService()); cred.addToken(jhToken.getService(), jhToken); }
protected void serviceStart() throws Exception { Configuration conf = new XLearningConfiguration(); YarnRPC rpc = YarnRPC.create(conf); initializeWebApp(conf); InetSocketAddress address = conf.getSocketAddr( XLearningConfiguration.XLEARNING_HISTORY_BIND_HOST, XLearningConfiguration.XLEARNING_HISTORY_ADDRESS, conf.get(XLearningConfiguration.XLEARNING_HISTORY_ADDRESS, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_ADDRESS), conf.getInt(XLearningConfiguration.XLEARNING_HISTORY_PORT, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_PORT)); server = rpc.getServer(HSClientProtocol.class, protocolHandler, address, conf, jhsDTSecretManager, conf.getInt(XLearningConfiguration.XLEARNING_HISTORY_CLIENT_THREAD_COUNT, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_CLIENT_THREAD_COUNT)); // Enable service authorization? if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { server.refreshServiceAcl(conf, new ClientHSPolicyProvider()); } server.start(); this.bindAddress = conf.updateConnectAddr(XLearningConfiguration.XLEARNING_HISTORY_BIND_HOST, XLearningConfiguration.XLEARNING_HISTORY_ADDRESS, conf.get(XLearningConfiguration.XLEARNING_HISTORY_ADDRESS, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_ADDRESS), server.getListenerAddress()); LOG.info("Instantiated HistoryClientService at " + this.bindAddress); super.serviceStart(); }
@Test public void testRMStartWithDecommissionedNode() throws Exception { String excludeFile = "excludeFile"; createExcludeFile(excludeFile); YarnConfiguration conf = new YarnConfiguration(); conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, excludeFile); MockRM rm = new MockRM(conf) { protected ClientRMService createClientRMService() { return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, this.getRMContext().getRMDelegationTokenSecretManager()); }; }; rm.start(); YarnRPC rpc = YarnRPC.create(conf); InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); LOG.info("Connecting to ResourceManager at " + rmAddress); ApplicationClientProtocol client = (ApplicationClientProtocol) rpc .getProxy(ApplicationClientProtocol.class, rmAddress, conf); // Make call GetClusterNodesRequest request = GetClusterNodesRequest.newInstance(EnumSet.allOf(NodeState.class)); List<NodeReport> nodeReports = client.getClusterNodes(request).getNodeReports(); Assert.assertEquals(1, nodeReports.size()); rm.stop(); rpc.stopProxy(client, conf); new File(excludeFile).delete(); }
@Test public void testUnknownCall() { Configuration conf = new Configuration(); conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class .getName()); YarnRPC rpc = YarnRPC.create(conf); String bindAddr = "localhost:0"; InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr); Server server = rpc.getServer(ContainerManagementProtocol.class, new DummyContainerManager(), addr, conf, null, 1); server.start(); // Any unrelated protocol would do ApplicationClientProtocol proxy = (ApplicationClientProtocol) rpc.getProxy( ApplicationClientProtocol.class, NetUtils.getConnectAddress(server), conf); try { proxy.getNewApplication(Records .newRecord(GetNewApplicationRequest.class)); Assert.fail("Excepted RPC call to fail with unknown method."); } catch (YarnException e) { Assert.assertTrue(e.getMessage().matches( "Unknown method getNewApplication called on.*" + "org.apache.hadoop.yarn.proto.ApplicationClientProtocol" + "\\$ApplicationClientProtocolService\\$BlockingInterface protocol.")); } catch (Exception e) { e.printStackTrace(); } }
@Override public T run() { return (T) rpc.getProxy(protocol, serverAddress, conf); } });
public YarnClientHelper(Configuration conf) { // Set up the configuration and RPC this.conf = conf; this.rpc = YarnRPC.create(conf); }
public Server getServer(Class protocol, Object instance, InetSocketAddress addr, Configuration conf, SecretManager<? extends TokenIdentifier> secretManager, int numHandlers) { return getServer(protocol, instance, addr, conf, secretManager, numHandlers, null); }
private boolean tryCloseProxy( ContainerManagementProtocolProxyData proxy) { proxy.activeCallers--; if (proxy.scheduledForClose && proxy.activeCallers < 0) { LOG.info("Closing proxy : " + proxy.containerManagerBindAddr); cmProxy.remove(proxy.containerManagerBindAddr); try { rpc.stopProxy(proxy.getContainerManagementProtocol(), conf); } finally { this.notifyAll(); } return true; } return false; }
rm.start(); YarnRPC rpc = YarnRPC.create(conf); InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); LOG.info("Connecting to ResourceManager at " + rmAddress); ApplicationClientProtocol client = (ApplicationClientProtocol) rpc .getProxy(ApplicationClientProtocol.class, rmAddress, conf); rpc.stopProxy(client, conf); new File(excludeFile).delete();
@Test public void testUnknownCall() { Configuration conf = new Configuration(); conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class .getName()); YarnRPC rpc = YarnRPC.create(conf); String bindAddr = "localhost:0"; InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr); Server server = rpc.getServer(ContainerManagementProtocol.class, new DummyContainerManager(), addr, conf, null, 1); server.start(); // Any unrelated protocol would do ApplicationClientProtocol proxy = (ApplicationClientProtocol) rpc.getProxy( ApplicationClientProtocol.class, NetUtils.getConnectAddress(server), conf); try { proxy.getNewApplication(Records .newRecord(GetNewApplicationRequest.class)); Assert.fail("Excepted RPC call to fail with unknown method."); } catch (YarnException e) { Assert.assertTrue(e.getMessage().matches( "Unknown method getNewApplication called on.*" + "org.apache.hadoop.yarn.proto.ApplicationClientProtocol" + "\\$ApplicationClientProtocolService\\$BlockingInterface protocol.")); } catch (Exception e) { e.printStackTrace(); } }
@Override public MRClientProtocol run() { return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class, hsAddress, conf); } });
/** * Constructor, connect to Resource Manager * * @throws IOException */ public ApplicationMaster() throws IOException { super(ApplicationMaster.class.getName()); // Set up the configuration and RPC conf = new MPIConfiguration(); rpc = YarnRPC.create(conf); dfs = FileSystem.get(conf); }
public Server getServer(Class protocol, Object instance, InetSocketAddress addr, Configuration conf, SecretManager<? extends TokenIdentifier> secretManager, int numHandlers) { return getServer(protocol, instance, addr, conf, secretManager, numHandlers, null); }
private boolean tryCloseProxy( ContainerManagementProtocolProxyData proxy) { proxy.activeCallers--; if (proxy.scheduledForClose && proxy.activeCallers < 0) { if (LOG.isDebugEnabled()) { LOG.debug("Closing proxy : " + proxy.containerManagerBindAddr); } cmProxy.remove(proxy.containerManagerBindAddr); try { rpc.stopProxy(proxy.getContainerManagementProtocol(), conf); } finally { this.notifyAll(); } return true; } return false; }
Server createServer() { Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); if (UserGroupInformation.isSecurityEnabled()) { secretManager = new LocalizerTokenSecretManager(); } Server server = rpc.getServer(LocalizationProtocol.class, this, localizationServerAddress, conf, secretManager, conf.getInt(YarnConfiguration.NM_LOCALIZER_CLIENT_THREAD_COUNT, YarnConfiguration.DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT)); // Enable service authorization? if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { server.refreshServiceAcl(conf, NMPolicyProvider.getInstance()); } return server; }
YarnRPC rpc = YarnRPC.create(conf); InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); LOG.info("Connecting to ResourceManager at " + rmAddress); ApplicationClientProtocol client = (ApplicationClientProtocol) rpc.getProxy( ApplicationClientProtocol.class, rmAddress, conf); Assert.assertTrue(nodeToLabels.get(node2).containsAll(Arrays.asList("y"))); rpc.stopProxy(client, conf); rm.close();
conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class .getName()); YarnRPC rpc = YarnRPC.create(conf); String bindAddr = "localhost:0"; InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr); Server server = rpc.getServer(ContainerManagementProtocol.class, new DummyContainerManager(), addr, conf, null, 1); server.start(); ApplicationClientProtocol proxy = (ApplicationClientProtocol) rpc.getProxy( ApplicationClientProtocol.class, NetUtils.getConnectAddress(server), conf);
@Override public T run() { return (T) rpc.getProxy(protocol, serverAddress, conf); } });
public Hadoop20YarnAMClient(Configuration conf) { super(ApplicationConstants.AM_CONTAINER_ID_ENV); this.amrmClient = new AMRMClientImpl(containerId.getApplicationAttemptId()); this.amrmClient.init(conf); this.nmClient = new Hadoop20YarnNMClient(YarnRPC.create(conf), conf); }