/** * returns the list of all resourcemanager ids for the given configuration. */ @Override protected Collection<String> getTargetIds(String targetNodeToActivate) { return HAUtil.getRMHAIds(getConf()); }
/** * returns the list of all resourcemanager ids for the given configuration. */ @Override protected Collection<String> getTargetIds(String targetNodeToActivate) { return HAUtil.getRMHAIds(getConf()); }
public static String getResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf, Policy httpPolicy) { String rmId = null; if (HAUtil.isHAEnabled(conf)) { // If HA enabled, pick one of the RM-IDs and rely on redirect to go to // the Active RM rmId = (String) HAUtil.getRMHAIds(conf).toArray()[0]; } return getResolvedRemoteRMWebAppURLWithoutScheme(conf, httpPolicy, rmId); }
private static void verifyAndSetCurrentRMHAId(Configuration conf) { String rmId = getRMHAId(conf); if (rmId == null) { StringBuilder msg = new StringBuilder(); msg.append("Can not find valid RM_HA_ID. None of "); for (String id : conf .getTrimmedStringCollection(YarnConfiguration.RM_HA_IDS)) { msg.append(addSuffix(YarnConfiguration.RM_ADDRESS, id)).append(" "); } msg.append(" are matching" + " the local address OR " + YarnConfiguration.RM_HA_ID + " is not" + " specified in HA Configuration"); throwBadConfigurationException(msg.toString()); } else { Collection<String> ids = getRMHAIds(conf); if (!ids.contains(rmId)) { throwBadConfigurationException( getRMHAIdNeedToBeIncludedMessage(ids.toString(), rmId)); } } conf.set(YarnConfiguration.RM_HA_ID, rmId); }
private static void verifyAndSetCurrentRMHAId(Configuration conf) { String rmId = getRMHAId(conf); if (rmId == null) { StringBuilder msg = new StringBuilder(); msg.append("Can not find valid RM_HA_ID. None of "); for (String id : conf .getTrimmedStringCollection(YarnConfiguration.RM_HA_IDS)) { msg.append(addSuffix(YarnConfiguration.RM_ADDRESS, id) + " "); } msg.append(" are matching" + " the local address OR " + YarnConfiguration.RM_HA_ID + " is not" + " specified in HA Configuration"); throwBadConfigurationException(msg.toString()); } else { Collection<String> ids = getRMHAIds(conf); if (!ids.contains(rmId)) { throwBadConfigurationException( getRMHAIdNeedToBeIncludedMessage(ids.toString(), rmId)); } } conf.set(YarnConfiguration.RM_HA_ID, rmId); }
@Override protected HAServiceTarget resolveTarget(String rmId) { Collection<String> rmIds = HAUtil.getRMHAIds(getConf()); if (!rmIds.contains(rmId)) { StringBuilder msg = new StringBuilder(); msg.append(rmId + " is not a valid serviceId. It should be one of "); for (String id : rmIds) { msg.append(id + " "); } throw new IllegalArgumentException(msg.toString()); } try { YarnConfiguration conf = new YarnConfiguration(getConf()); conf.set(YarnConfiguration.RM_HA_ID, rmId); return new RMHAServiceTarget(conf); } catch (IllegalArgumentException iae) { throw new YarnRuntimeException("Could not connect to " + rmId + "; the configuration for it might be missing"); } catch (IOException ioe) { throw new YarnRuntimeException( "Could not connect to RM HA Admin for node " + rmId); } }
@Unstable public static Text getTokenService(Configuration conf, String address, String defaultAddr, int defaultPort) { if (HAUtil.isHAEnabled(conf)) { // Build a list of service addresses to form the service name ArrayList<String> services = new ArrayList<String>(); YarnConfiguration yarnConf = new YarnConfiguration(conf); for (String rmId : HAUtil.getRMHAIds(conf)) { // Set RM_ID to get the corresponding RM_ADDRESS yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); services.add(SecurityUtil.buildTokenService( yarnConf.getSocketAddr(address, defaultAddr, defaultPort)) .toString()); } return new Text(Joiner.on(',').join(services)); } // Non-HA case - no need to set RM_ID return SecurityUtil.buildTokenService(conf.getSocketAddr(address, defaultAddr, defaultPort)); } }
@Override protected HAServiceTarget resolveTarget(String rmId) { Collection<String> rmIds = HAUtil.getRMHAIds(getConf()); if (!rmIds.contains(rmId)) { StringBuilder msg = new StringBuilder(); msg.append(rmId + " is not a valid serviceId. It should be one of "); for (String id : rmIds) { msg.append(id + " "); } throw new IllegalArgumentException(msg.toString()); } try { YarnConfiguration conf = new YarnConfiguration(getConf()); conf.set(YarnConfiguration.RM_HA_ID, rmId); return new RMHAServiceTarget(conf); } catch (IllegalArgumentException iae) { throw new YarnRuntimeException("Could not connect to " + rmId + "; the configuration for it might be missing"); } catch (IOException ioe) { throw new YarnRuntimeException( "Could not connect to RM HA Admin for node " + rmId); } }
@Unstable public static Text getTokenService(Configuration conf, String address, String defaultAddr, int defaultPort) { if (HAUtil.isHAEnabled(conf)) { // Build a list of service addresses to form the service name ArrayList<String> services = new ArrayList<String>(); YarnConfiguration yarnConf = new YarnConfiguration(conf); for (String rmId : HAUtil.getRMHAIds(conf)) { // Set RM_ID to get the corresponding RM_ADDRESS yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); services.add(SecurityUtil.buildTokenService( yarnConf.getSocketAddr(address, defaultAddr, defaultPort)) .toString()); } return new Text(Joiner.on(',').join(services)); } // Non-HA case - no need to set RM_ID return SecurityUtil.buildTokenService(conf.getSocketAddr(address, defaultAddr, defaultPort)); } }
@InterfaceStability.Unstable public static Text getTokenService(Configuration conf, String address, String defaultAddr, int defaultPort) { if (HAUtil.isHAEnabled(conf)) { // Build a list of service addresses to form the service name ArrayList<String> services = new ArrayList<String>(); YarnConfiguration yarnConf = new YarnConfiguration(conf); for (String rmId : HAUtil.getRMHAIds(conf)) { // Set RM_ID to get the corresponding RM_ADDRESS yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); services.add(SecurityUtil.buildTokenService( yarnConf.getSocketAddr(address, defaultAddr, defaultPort)) .toString()); } return new Text(Joiner.on(',').join(services)); } // Non-HA case - no need to set RM_ID return SecurityUtil.buildTokenService(conf.getSocketAddr(address, defaultAddr, defaultPort)); } }
@Unstable public static Text getTokenService(Configuration conf, String address, String defaultAddr, int defaultPort) { if (HAUtil.isHAEnabled(conf)) { // Build a list of service addresses to form the service name ArrayList<String> services = new ArrayList<String>(); YarnConfiguration yarnConf = new YarnConfiguration(conf); for (String rmId : HAUtil.getRMHAIds(conf)) { // Set RM_ID to get the corresponding RM_ADDRESS yarnConf.set(YarnConfiguration.RM_HA_ID, rmId); services.add(SecurityUtil.buildTokenService( yarnConf.getSocketAddr(address, defaultAddr, defaultPort)) .toString()); } return new Text(Joiner.on(',').join(services)); } // Non-HA case - no need to set RM_ID return SecurityUtil.buildTokenService(conf.getSocketAddr(address, defaultAddr, defaultPort)); } }
@Override public void init(Configuration configuration, RMProxy<T> rmProxy, Class<T> protocol) { this.rmProxy = rmProxy; this.protocol = protocol; this.rmProxy.checkAllowedProtocols(this.protocol); this.conf = new YarnConfiguration(configuration); Collection<String> rmIds = HAUtil.getRMHAIds(conf); this.rmServiceIds = rmIds.toArray(new String[rmIds.size()]); conf.set(YarnConfiguration.RM_HA_ID, rmServiceIds[currentProxyIndex]); conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES)); conf.setInt(CommonConfigurationKeysPublic. IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS, YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS)); }
private URL getClusterUrl() throws Exception { URL url = null; Configuration conf = getConf(); if (HAUtil.isHAEnabled(conf)) { Collection<String> haids = HAUtil.getRMHAIds(conf); for (String rmhid : haids) { try { url = getHAClusterUrl(conf, rmhid); if (isActive(url)) { break; } } catch (ConnectException e) { // ignore and try second one when one of RM is down } } } else { url = new URL( WebAppUtils.getRMWebAppURLWithScheme(conf) + CLUSTER_INFO_URL); } return url; }
@Override public void init(Configuration configuration, RMProxy<T> rmProxy, Class<T> protocol) { this.rmProxy = rmProxy; this.protocol = protocol; this.rmProxy.checkAllowedProtocols(this.protocol); this.conf = new YarnConfiguration(configuration); Collection<String> rmIds = HAUtil.getRMHAIds(conf); this.rmServiceIds = rmIds.toArray(new String[rmIds.size()]); conf.set(YarnConfiguration.RM_HA_ID, rmServiceIds[currentProxyIndex]); conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES)); conf.setInt(CommonConfigurationKeysPublic. IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS, YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS)); }
private URL getClusterUrl() throws Exception { URL url = null; Configuration conf = getConf(); if (HAUtil.isHAEnabled(conf)) { Collection<String> haids = HAUtil.getRMHAIds(conf); for (String rmhid : haids) { try { url = getHAClusterUrl(conf, rmhid); if (isActive(url)) { break; } } catch (ConnectException e) { // ignore and try second one when one of RM is down } } } else { url = new URL( WebAppUtils.getRMWebAppURLWithScheme(conf) + CLUSTER_INFO_URL); } return url; }
@Override public void init(Configuration configuration, RMProxy<T> rmProxy, Class<T> protocol) { this.rmProxy = rmProxy; this.protocol = protocol; this.rmProxy.checkAllowedProtocols(this.protocol); this.conf = new YarnConfiguration(configuration); Collection<String> rmIds = HAUtil.getRMHAIds(conf); this.rmServiceIds = rmIds.toArray(new String[rmIds.size()]); conf.set(YarnConfiguration.RM_HA_ID, rmServiceIds[currentProxyIndex]); conf.setInt(CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES)); conf.setInt(CommonConfigurationKeysPublic. IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, conf.getInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS, YarnConfiguration.DEFAULT_CLIENT_FAILOVER_RETRIES_ON_SOCKET_TIMEOUTS)); }
@Test public void testGetRMServiceId() throws Exception { conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); Collection<String> rmhaIds = HAUtil.getRMHAIds(conf); assertEquals(2, rmhaIds.size()); String[] ids = rmhaIds.toArray(new String[0]); assertEquals(RM1_NODE_ID, ids[0]); assertEquals(RM2_NODE_ID, ids[1]); }
private Configuration createHARMConf( String rmIds, String rmId, int adminPort) { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true); conf.set(YarnConfiguration.RM_HA_IDS, rmIds); conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.set(YarnConfiguration.RM_STORE, ZKRMStateStore.class.getName()); conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort); conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, ZK_TIMEOUT_MS); conf.set(YarnConfiguration.RM_HA_ID, rmId); conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, "localhost:0"); for (String rpcAddress : YarnConfiguration.getServiceAddressConfKeys(conf)) { for (String id : HAUtil.getRMHAIds(conf)) { conf.set(HAUtil.addSuffix(rpcAddress, id), "localhost:0"); } } conf.set(HAUtil.addSuffix(YarnConfiguration.RM_ADMIN_ADDRESS, rmId), "localhost:" + adminPort); return conf; }
@Test public void testGetRMServiceId() throws Exception { conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); Collection<String> rmhaIds = HAUtil.getRMHAIds(conf); assertEquals(2, rmhaIds.size()); String[] ids = rmhaIds.toArray(new String[0]); assertEquals(RM1_NODE_ID, ids[0]); assertEquals(RM2_NODE_ID, ids[1]); }
@Test public void testGetRMServiceId() throws Exception { conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID); Collection<String> rmhaIds = HAUtil.getRMHAIds(conf); assertEquals(2, rmhaIds.size()); String[] ids = rmhaIds.toArray(new String[0]); assertEquals(RM1_NODE_ID, ids[0]); assertEquals(RM2_NODE_ID, ids[1]); }