@Override public List<KeyRange> getLocalKeyPartition() throws BackendException { List<KeyRange> result = new LinkedList<KeyRange>(); HTable table = null; try { ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0); table = new HTable(hconf, tableName); Map<KeyRange, ServerName> normed = normalizeKeyBounds(table.getRegionLocations()); for (Map.Entry<KeyRange, ServerName> e : normed.entrySet()) { if (NetworkUtil.isLocalConnection(e.getValue().getHostname())) { result.add(e.getKey()); logger.debug("Found local key/row partition {} on host {}", e.getKey(), e.getValue()); } else { logger.debug("Discarding remote {}", e.getValue()); } } } catch (MasterNotRunningException e) { logger.warn("Unexpected MasterNotRunningException", e); } catch (ZooKeeperConnectionException e) { logger.warn("Unexpected ZooKeeperConnectionException", e); } catch (IOException e) { logger.warn("Unexpected IOException", e); } finally { IOUtils.closeQuietly(table); } return result; }
Set<HRegionInfo> tableRegionInfos = table.getRegionLocations().keySet(); Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
HTable htable = (HTable)conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(tableName); htable.clearRegionCache(); int nRegions = htable.getRegionLocations().size(); admin.split(tableName, ByteUtil.concat(Bytes.toBytes(tenantId), Bytes.toBytes("00A" + Character.valueOf((char)('3' + nextRunCount()))+ ts))); // vary split point with test run int retryCount = 0; retryCount++; } while (retryCount < 10 && htable.getRegionLocations().size() == nRegions); assertNotEquals(nRegions,htable.getRegionLocations().size());
private void split(HTable table, byte[] splitPoint) throws IOException, InterruptedException, DecoderException { int regionCountPrev = table.getRegionLocations().size(); admin.split(table.getTableName(), splitPoint); for (int j = 0; j < Constant.TRY_MAX; j++) { int regionCountNow = table.getRegionLocations().size(); if (regionCountPrev < regionCountNow) { break; } else { Thread.sleep(Constant.WAIT_INTERVAL_MS); } } }
/** * Sets the number of reduce tasks for the given job configuration to the * number of regions the given table has. * * @param table * The table to get the region count for. * @param job * The current job to adjust. * @throws IOException * When retrieving the table details fails. */ public static void setNumReduceTasks(String table, Job job) throws IOException { HTable outputTable = new HTable(job.getConfiguration(), table); int regions = outputTable.getRegionLocations().size(); job.setNumReduceTasks(regions); }
private static void getRegionLocations(HBaseAdmin admin, String tableName) throws IOException { try (HTable table = new HTable(admin.getConfiguration(), tableName)) { regionLocations.putAll(table.getRegionLocations()); cachedTableNames.add(tableName); } }
private void refreshRegionInfo(String tableName) throws IOException { System.out.println("in refreshRegionInfo "); Map<HRegionInfo, ServerName> regionMap = hTable.getRegionLocations(); synchronized (regionStartKeys) { synchronized (regionStartKeyRegionNameMap) { regionStartKeys = new String[regionMap.size()]; int index = 0; String startKey = null; regionStartKeyRegionNameMap.clear(); for (HRegionInfo regionInfo : regionMap.keySet()) { startKey = new String(regionInfo.getStartKey()); regionStartKeyRegionNameMap.put(startKey, regionInfo.getRegionNameAsString()); regionStartKeys[index] = startKey; index++; } Arrays.sort(regionStartKeys); regionCheckTime = System.currentTimeMillis(); } } } }
/** * Ensures that the given number of reduce tasks for the given job * configuration does not exceed the number of regions for the given table. * * @param table * The table to get the region count for. * @param job * The current job to adjust. * @throws IOException * When retrieving the table details fails. */ public static void limitNumReduceTasks(String table, Job job) throws IOException { HTable outputTable = new HTable(job.getConfiguration(), table); int regions = outputTable.getRegionLocations().size(); if (job.getNumReduceTasks() > regions) job.setNumReduceTasks(regions); }
private NavigableMap<HRegionInfo, ServerName> getRegionLocations(String tableName, HBaseAdmin admin) throws IOException { try (HTable table = new HTable(admin.getConfiguration(), tableName)) { return table.getRegionLocations(); } } };
/** * Gets the starting and ending row keys for every region in the currently * open table. * <p> * This is mainly useful for the MapReduce integration. * @return Pair of arrays of region starting and ending row keys * @throws IOException if a remote or network exception occurs */ public Pair<byte[][],byte[][]> getStartEndKeys() throws IOException { NavigableMap<HRegionInfo, ServerName> regions = getRegionLocations(); final List<byte[]> startKeyList = new ArrayList<byte[]>(regions.size()); final List<byte[]> endKeyList = new ArrayList<byte[]>(regions.size()); for (HRegionInfo region : regions.keySet()) { startKeyList.add(region.getStartKey()); endKeyList.add(region.getEndKey()); } return new Pair<byte [][], byte [][]>( startKeyList.toArray(new byte[startKeyList.size()][]), endKeyList.toArray(new byte[endKeyList.size()][])); }
/** * Get the Cassandra Fiji Partitions for the given cluster. * * @param htable An open connection to the HBase table. * @return The collection of Fiji partitions. * @throws IOException if a remote or network exception occurs. */ public static Collection<HBaseFijiPartition> getPartitions( final HTable htable ) throws IOException { final ImmutableList.Builder<HBaseFijiPartition> partitions = ImmutableList.builder(); NavigableMap<HRegionInfo, ServerName> regionLocations = htable.getRegionLocations(); for (Map.Entry<HRegionInfo, ServerName> regionLocation : regionLocations.entrySet()) { partitions.add( new HBaseFijiPartition( InetAddress.getByName(regionLocation.getValue().getHostname()), regionLocation.getKey().getStartKey(), regionLocation.getKey().getEndKey())); } return partitions.build(); } }
/** * Gets all the regions and their address for this table. * @return A map of HRegionInfo with it's server address * @throws IOException if a remote or network exception occurs * @deprecated Use {@link #getRegionLocations()} or {@link #getStartEndKeys()} */ public Map<HRegionInfo, HServerAddress> getRegionsInfo() throws IOException { final Map<HRegionInfo, HServerAddress> regionMap = new TreeMap<HRegionInfo, HServerAddress>(); final Map<HRegionInfo, ServerName> regionLocations = getRegionLocations(); for (Map.Entry<HRegionInfo, ServerName> entry : regionLocations.entrySet()) { HServerAddress server = new HServerAddress(); ServerName serverName = entry.getValue(); if (serverName != null && serverName.getHostAndPort() != null) { server = new HServerAddress(Addressing.createInetSocketAddressFromHostAndPortStr( serverName.getHostAndPort())); } regionMap.put(entry.getKey(), server); } return regionMap; }
private NavigableMap<HRegionInfo, ServerName> getRegionLocations(String table) throws IOException { long startTimestamp = System.currentTimeMillis(); Util.printVerboseMessage(args, Util.getMethodName() + " - start"); NavigableMap<HRegionInfo, ServerName> result = regionLocations.get(table); if (result == null) { try (HTable htable = new HTable(admin.getConfiguration(), table)) { result = htable.getRegionLocations(); regionLocations.put(table, result); } } Util.printVerboseMessage(args, Util.getMethodName() + " - end", startTimestamp); return result; }
public static boolean isMoved(HBaseAdmin admin, String tableName, String regionName, String serverNameTarget) { try (HTable table = new HTable(admin.getConfiguration(), tableName)) { NavigableMap<HRegionInfo, ServerName> regionLocations = table.getRegionLocations(); for (Map.Entry<HRegionInfo, ServerName> regionLocation : regionLocations.entrySet()) { if (regionLocation.getKey().getEncodedName().equals(regionName)) { return regionLocation.getValue().getServerName().equals(serverNameTarget); } } if (!existsRegion(regionName, regionLocations.keySet())) return true; // skip moving } catch (IOException e) { return false; } return false; }
@Override public List<KeyRange> getLocalKeyPartition() throws BackendException { List<KeyRange> result = new LinkedList<>(); TableMask table = null; try { ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0); table = cnx.getTable(tableName); HTable hTable = (HTable)table.getTableObject(); Map<KeyRange, ServerName> normed = normalizeKeyBounds(hTable.getRegionLocations()); for (Map.Entry<KeyRange, ServerName> e : normed.entrySet()) { if (NetworkUtil.isLocalConnection(e.getValue().getHostname())) { result.add(e.getKey()); logger.debug("Found local key/row partition {} on host {}", e.getKey(), e.getValue()); } else { logger.debug("Discarding remote {}", e.getValue()); } } } catch (MasterNotRunningException e) { logger.warn("Unexpected MasterNotRunningException", e); } catch (ZooKeeperConnectionException e) { logger.warn("Unexpected ZooKeeperConnectionException", e); } catch (IOException e) { logger.warn("Unexpected IOException", e); } finally { IOUtils.closeQuietly(table); } return result; }
@Override public List<KeyRange> getLocalKeyPartition() throws BackendException { List<KeyRange> result = new LinkedList<KeyRange>(); TableMask table = null; try { ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0); table = cnx.getTable(tableName); HTable hTable = (HTable)table.getTableObject(); Map<KeyRange, ServerName> normed = normalizeKeyBounds(hTable.getRegionLocations()); for (Map.Entry<KeyRange, ServerName> e : normed.entrySet()) { if (NetworkUtil.isLocalConnection(e.getValue().getHostname())) { result.add(e.getKey()); logger.debug("Found local key/row partition {} on host {}", e.getKey(), e.getValue()); } else { logger.debug("Discarding remote {}", e.getValue()); } } } catch (MasterNotRunningException e) { logger.warn("Unexpected MasterNotRunningException", e); } catch (ZooKeeperConnectionException e) { logger.warn("Unexpected ZooKeeperConnectionException", e); } catch (IOException e) { logger.warn("Unexpected IOException", e); } finally { IOUtils.closeQuietly(table); } return result; }
protected ArrayList<HRegionInfo> getRegionInfoList(String tableName) throws IOException { Set<HRegionInfo> hRegionInfoSet = new TreeSet<>(); try (HTable table = (HTable) hConnection.getTable(tableName)) { hRegionInfoSet.addAll(table.getRegionLocations().keySet()); } return new ArrayList<>(hRegionInfoSet); }
private long getWriteRequestCountActual(String tableName) throws IOException { long writeRequestCountActual; try (HTable table = (HTable) hConnection.getTable(tableName)) { writeRequestCountActual = 0; NavigableMap<HRegionInfo, ServerName> regionLocations = table.getRegionLocations(); for (Map.Entry<HRegionInfo, ServerName> entry : regionLocations.entrySet()) { ServerLoad serverLoad = admin.getClusterStatus().getLoad(entry.getValue()); for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) { if (Arrays.equals(entry.getKey().getRegionName(), regionLoad.getName())) writeRequestCountActual += regionLoad.getWriteRequestsCount(); } } } return writeRequestCountActual; }
protected void waitForSplitting(String tableName, int regionCount) throws IOException, InterruptedException { int regionCountActual = 0; for (int i = 0; i < MAX_WAIT_ITERATION; i++) { try (HTable table = (HTable) hConnection.getTable(tableName)) { regionCountActual = 0; NavigableMap<HRegionInfo, ServerName> regionLocations = table.getRegionLocations(); for (Map.Entry<HRegionInfo, ServerName> entry : regionLocations.entrySet()) { ServerLoad serverLoad = admin.getClusterStatus().getLoad(entry.getValue()); for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) { if (Arrays.equals(entry.getKey().getRegionName(), regionLoad.getName())) regionCountActual++; } } if (regionCountActual == regionCount) { return; } } catch (Throwable ignore) { } Thread.sleep(WAIT_INTERVAL); } Assert.assertEquals("TestBase.waitForSplitting - failed - ", regionCount, regionCountActual); }
private long getWriteRequestCountActual(String tableName, ServerName serverName) throws IOException { long writeRequestCountActual; try (HTable table = (HTable) hConnection.getTable(tableName)) { writeRequestCountActual = 0; NavigableMap<HRegionInfo, ServerName> regionLocations = table.getRegionLocations(); for (Map.Entry<HRegionInfo, ServerName> entry : regionLocations.entrySet()) { if (serverName.equals(entry.getValue())) { ServerLoad serverLoad = admin.getClusterStatus().getLoad(entry.getValue()); for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) { if (Arrays.equals(entry.getKey().getRegionName(), regionLoad.getName())) writeRequestCountActual += regionLoad.getWriteRequestsCount(); } } } } Long aLong = getWriteRequestMetric(tableName, serverName); return writeRequestCountActual - aLong; }