private List<ServerName> filterOfflineServers(RSGroupInfo RSGroupInfo, List<ServerName> onlineServers) { if (RSGroupInfo != null) { return filterServers(RSGroupInfo.getServers(), onlineServers); } else { LOG.warn("RSGroup Information found to be null. Some regions might be unassigned."); return Collections.emptyList(); } }
if(rsGroupInfo.getServers().contains(serverName.getAddress())) { serverMap.put(serverName, Collections.emptyList());
static RSGroupProtos.RSGroupInfo toProtoGroupInfo(RSGroupInfo pojo) { List<HBaseProtos.TableName> tables = new ArrayList<>(pojo.getTables().size()); for(TableName arg: pojo.getTables()) { tables.add(ProtobufUtil.toProtoTableName(arg)); } List<HBaseProtos.ServerName> hostports = new ArrayList<>(pojo.getServers().size()); for(Address el: pojo.getServers()) { hostports.add(HBaseProtos.ServerName.newBuilder() .setHostName(el.getHostname()) .setPort(el.getPort()) .build()); } return RSGroupProtos.RSGroupInfo.newBuilder().setName(pojo.getName()) .addAllServers(hostports) .addAllTables(tables).build(); } }
@Override public synchronized Set<Address> moveServers(Set<Address> servers, String srcGroup, String dstGroup) throws IOException { RSGroupInfo src = getRSGroupInfo(srcGroup); RSGroupInfo dst = getRSGroupInfo(dstGroup); // If destination is 'default' rsgroup, only add servers that are online. If not online, drop // it. If not 'default' group, add server to 'dst' rsgroup EVEN IF IT IS NOT online (could be a // rsgroup of dead servers that are to come back later). Set<Address> onlineServers = dst.getName().equals(RSGroupInfo.DEFAULT_GROUP) ? getOnlineServers(this.masterServices) : null; for (Address el : servers) { src.removeServer(el); if (onlineServers != null) { if (!onlineServers.contains(el)) { if (LOG.isDebugEnabled()) { LOG.debug("Dropping " + el + " during move-to-default rsgroup because not online"); } continue; } } dst.addServer(el); } Map<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap); newGroupMap.put(src.getName(), src); newGroupMap.put(dst.getName(), dst); flushConfig(newGroupMap); return dst.getServers(); }
/** * Test BOGUS_SERVER_NAME among groups do not overwrite each other. */ @Test public void testRoundRobinAssignment() throws Exception { List<ServerName> onlineServers = new ArrayList<ServerName>(servers.size()); onlineServers.addAll(servers); List<RegionInfo> regions = randomRegions(25); int bogusRegion = 0; for(RegionInfo region : regions){ String group = tableMap.get(region.getTable()); if("dg3".equals(group) || "dg4".equals(group)){ bogusRegion++; } } Set<Address> offlineServers = new HashSet<Address>(); offlineServers.addAll(groupMap.get("dg3").getServers()); offlineServers.addAll(groupMap.get("dg4").getServers()); for(Iterator<ServerName> it = onlineServers.iterator(); it.hasNext();){ ServerName server = it.next(); Address address = server.getAddress(); if(offlineServers.contains(address)){ it.remove(); } } Map<ServerName, List<RegionInfo>> assignments = loadBalancer .roundRobinAssignment(regions, onlineServers); assertEquals(bogusRegion, assignments.get(LoadBalancer.BOGUS_SERVER_NAME).size()); } }
@Test public void testGroupInfoMultiAccessing() throws Exception { RSGroupInfoManager manager = rsGroupAdminEndpoint.getGroupInfoManager(); RSGroupInfo defaultGroup = manager.getRSGroup("default"); // getRSGroup updates default group's server list // this process must not affect other threads iterating the list Iterator<Address> it = defaultGroup.getServers().iterator(); manager.getRSGroup("default"); it.next(); }
protected String printStats( ArrayListMultimap<String, ServerAndLoad> groupBasedLoad) { StringBuilder sb = new StringBuilder(); sb.append("\n"); for (String groupName : groupBasedLoad.keySet()) { sb.append("Stats for group: " + groupName); sb.append("\n"); sb.append(groupMap.get(groupName).getServers()); sb.append("\n"); List<ServerAndLoad> groupLoad = groupBasedLoad.get(groupName); int numServers = groupLoad.size(); int totalRegions = 0; sb.append("Per Server Load: \n"); for (ServerAndLoad sLoad : groupLoad) { sb.append("Server :" + sLoad.getServerName() + " Load : " + sLoad.getLoad() + "\n"); totalRegions += sLoad.getLoad(); } sb.append(" Group Statistics : \n"); float average = (float) totalRegions / numServers; int max = (int) Math.ceil(average); int min = (int) Math.floor(average); sb.append("[srvr=" + numServers + " rgns=" + totalRegions + " avg=" + average + " max=" + max + " min=" + min + "]"); sb.append("\n"); sb.append("==============================="); sb.append("\n"); } return sb.toString(); }
public RSGroupInfo addGroup(String groupName, int serverCount) throws IOException, InterruptedException { RSGroupInfo defaultInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.addRSGroup(groupName); Set<Address> set = new HashSet<>(); for(Address server: defaultInfo.getServers()) { if(set.size() == serverCount) { break; } set.add(server); } rsGroupAdmin.moveServers(set, groupName); RSGroupInfo result = rsGroupAdmin.getRSGroupInfo(groupName); return result; }
"the rsgroup can be removed."); int serverCount = rsGroupInfo.getServers().size(); if (serverCount > 0) { throw new ConstraintException("RSGroup " + name + " has " + serverCount +
boolean rsgroupHasServersOnline(TableDescriptor desc) throws IOException { String groupName; try { groupName = master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString()) .getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP); if (groupName == null) { groupName = RSGroupInfo.DEFAULT_GROUP; } } catch (MasterNotRunningException | PleaseHoldException e) { LOG.info("Master has not initialized yet; temporarily using default RSGroup '" + RSGroupInfo.DEFAULT_GROUP + "' for deploy of system table"); groupName = RSGroupInfo.DEFAULT_GROUP; } RSGroupInfo rsGroupInfo = groupAdminServer.getRSGroupInfo(groupName); if (rsGroupInfo == null) { throw new ConstraintException( "Default RSGroup (" + groupName + ") for this table's " + "namespace does not exist."); } for (ServerName onlineServer : master.getServerManager().createDestinationServersList()) { if (rsGroupInfo.getServers().contains(onlineServer.getAddress())) { return true; } } return false; }
@Test public void testRegionServerMove() throws IOException, InterruptedException { int initNumGroups = rsGroupAdmin.listRSGroups().size(); RSGroupInfo appInfo = addGroup(getGroupName(name.getMethodName()), 1); RSGroupInfo adminInfo = addGroup(getGroupName(name.getMethodName()), 1); RSGroupInfo dInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); Assert.assertEquals(initNumGroups + 2, rsGroupAdmin.listRSGroups().size()); assertEquals(1, adminInfo.getServers().size()); assertEquals(1, appInfo.getServers().size()); assertEquals(getNumServers() - 2, dInfo.getServers().size()); rsGroupAdmin.moveServers(appInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.removeRSGroup(appInfo.getName()); rsGroupAdmin.moveServers(adminInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.removeRSGroup(adminInfo.getName()); Assert.assertEquals(rsGroupAdmin.listRSGroups().size(), initNumGroups); }
@Test public void testBasicStartUp() throws IOException { RSGroupInfo defaultInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); assertEquals(4, defaultInfo.getServers().size()); // Assignment of root and meta regions. int count = master.getAssignmentManager().getRegionStates().getRegionAssignments().size(); // 2 meta, group assertEquals(2, count); }
public void removeGroup(String groupName) throws IOException { RSGroupInfo groupInfo = rsGroupAdmin.getRSGroupInfo(groupName); rsGroupAdmin.moveTables(groupInfo.getTables(), RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.moveServers(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.removeRSGroup(groupName); }
@Test public void testClearNotProcessedDeadServer() throws Exception { LOG.info("testClearNotProcessedDeadServer"); NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size(); RSGroupInfo appInfo = addGroup("deadServerGroup", 1); ServerName targetServer = getServerName(appInfo.getServers().iterator().next()); try { // stopping may cause an exception // due to the connection loss admin.stopRegionServer(targetServer.getAddress().toString()); NUM_DEAD_SERVERS++; } catch (Exception e) { } TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return cluster.getClusterMetrics().getDeadServerNames().size() == NUM_DEAD_SERVERS; } }); List<ServerName> notClearedServers = admin.clearDeadServers(Lists.newArrayList(targetServer)); assertEquals(1, notClearedServers.size()); }
NUM_DEAD_SERVERS = cluster.getClusterMetrics().getDeadServerNames().size(); ServerName targetServer = getServerName(newGroup.getServers().iterator().next()); try { assertFalse(cluster.getClusterMetrics().getLiveServerMetrics().containsKey(targetServer)); assertTrue(cluster.getClusterMetrics().getDeadServerNames().contains(targetServer)); assertTrue(newGroup.getServers().contains(targetServer.getAddress())); assertEquals(0, notClearedServers.size()); Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers(); assertFalse(newGroupServers.contains(targetServer.getAddress())); assertEquals(2, newGroupServers.size());
public void deleteGroups() throws IOException { RSGroupAdmin groupAdmin = new RSGroupAdminClient(TEST_UTIL.getConnection()); for(RSGroupInfo group: groupAdmin.listRSGroups()) { if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) { groupAdmin.moveTables(group.getTables(), RSGroupInfo.DEFAULT_GROUP); groupAdmin.moveServers(group.getServers(), RSGroupInfo.DEFAULT_GROUP); groupAdmin.removeRSGroup(group.getName()); } } }
protected ArrayListMultimap<String, ServerAndLoad> convertToGroupBasedMap( final Map<ServerName, List<RegionInfo>> serversMap) throws IOException { ArrayListMultimap<String, ServerAndLoad> loadMap = ArrayListMultimap .create(); for (RSGroupInfo gInfo : getMockedGroupInfoManager().listRSGroups()) { Set<Address> groupServers = gInfo.getServers(); for (Address hostPort : groupServers) { ServerName actual = null; for(ServerName entry: servers) { if(entry.getAddress().equals(hostPort)) { actual = entry; break; } } List<RegionInfo> regions = serversMap.get(actual); assertTrue("No load for " + actual, regions != null); loadMap.put(gInfo.getName(), new ServerAndLoad(actual, regions.size())); } } return loadMap; }
@Test public void testNamespaceCreateAndAssign() throws Exception { LOG.info("testNamespaceCreateAndAssign"); String nsName = tablePrefix + "_foo"; final TableName tableName = TableName.valueOf(nsName, tablePrefix + "_testCreateAndAssign"); RSGroupInfo appInfo = addGroup("appInfo", 1); admin.createNamespace(NamespaceDescriptor.create(nsName) .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "appInfo").build()); final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build(); admin.createTable(desc); // wait for created table to be assigned TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { return getTableRegionMap().get(desc.getTableName()) != null; } }); ServerName targetServer = getServerName(appInfo.getServers().iterator().next()); // verify it was assigned to the right group Assert.assertEquals(1, admin.getRegions(targetServer).size()); }
Iterator<Address> iterator = appInfo.getServers().iterator(); List<ServerName> serversToDecommission = new ArrayList<>(); ServerName targetServer = getServerName(iterator.next());
@Test public void testFailRemoveGroup() throws IOException, InterruptedException { int initNumGroups = rsGroupAdmin.listRSGroups().size(); addGroup("bar", 3); TEST_UTIL.createTable(tableName, Bytes.toBytes("f")); rsGroupAdmin.moveTables(Sets.newHashSet(tableName), "bar"); RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar"); // group is not empty therefore it should fail try { rsGroupAdmin.removeRSGroup(barGroup.getName()); fail("Expected remove group to fail"); } catch (IOException e) { } // group cannot lose all it's servers therefore it should fail try { rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP); fail("Expected move servers to fail"); } catch (IOException e) { } rsGroupAdmin.moveTables(barGroup.getTables(), RSGroupInfo.DEFAULT_GROUP); try { rsGroupAdmin.removeRSGroup(barGroup.getName()); fail("Expected move servers to fail"); } catch (IOException e) { } rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.removeRSGroup(barGroup.getName()); Assert.assertEquals(initNumGroups, rsGroupAdmin.listRSGroups().size()); }