table = new HTable(adm.getConfiguration(), tableName);
table = new HTable(adm.getConfiguration(), tableName);
table = new HTable(adm.getConfiguration(), tableName);
private void setup() throws IOException { long currentTIME = System.currentTimeMillis(); exportFolder = "/tmp/hbase-export/" + currentTIME + "/"; backupArchive = "/tmp/kylin_" + currentTIME + ".tar.gz"; KylinConfig.destoryInstance(); System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA); kylinConfig = KylinConfig.getInstanceFromEnv(); cli = kylinConfig.getCliCommandExecutor(); String metadataUrl = kylinConfig.getMetadataUrl(); // split TABLE@HBASE_URL int cut = metadataUrl.indexOf('@'); tableNameBase = metadataUrl.substring(0, cut); String hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1); HConnection conn = HBaseConnection.get(hbaseUrl); try { hbase = new HBaseAdmin(conn); config = hbase.getConfiguration(); allTables = hbase.listTables(); } catch (IOException e) { e.printStackTrace(); throw e; } }
public void setup() throws IOException { KylinConfig.destoryInstance(); System.setProperty(KylinConfig.KYLIN_CONF, AbstractKylinTestCase.SANDBOX_TEST_DATA); kylinConfig = KylinConfig.getInstanceFromEnv(); cli = kylinConfig.getCliCommandExecutor(); String metadataUrl = kylinConfig.getMetadataUrl(); // split TABLE@HBASE_URL int cut = metadataUrl.indexOf('@'); tableNameBase = metadataUrl.substring(0, cut); String hbaseUrl = cut < 0 ? metadataUrl : metadataUrl.substring(cut + 1); HConnection conn = HBaseConnection.get(hbaseUrl); try { hbase = new HBaseAdmin(conn); config = hbase.getConfiguration(); //allTables = hbase.listTables(); } catch (IOException e) { e.printStackTrace(); throw e; } uploadTarballToRemote(); }
@Test public void testCachedConnections() throws Exception { final String tableName = generateUniqueName(); final String index1Name = generateUniqueName(); final Connection conn = DriverManager.getConnection(getUrl()); final HBaseAdmin admin = getUtility().getHBaseAdmin(); final MiniHBaseCluster cluster = getUtility().getHBaseCluster(); final HRegionServer regionServer = cluster.getRegionServer(0); Configuration conf = admin.getConfiguration(); final int noOfOrgs = 20; final AtomicBoolean flag = new AtomicBoolean(); flag.set(false); // create table and indices String createTableSql = "CREATE TABLE " + tableName + "(org_id VARCHAR NOT NULL PRIMARY KEY, v1 INTEGER, v2 INTEGER, v3 INTEGER) VERSIONS=1 SPLIT ON ('" + ORG_PREFIX + "-" + noOfOrgs / 2 + "')"; conn.createStatement().execute(createTableSql); conn.createStatement().execute("CREATE INDEX " + index1Name + " ON " + tableName + "(v1)"); List<HRegionInfo> regions = admin.getTableRegions(TableName.valueOf(tableName)); final HRegionInfo regionInfo = regions.get(0); writeToTable(tableName, conn, noOfOrgs); int beforeRegionCloseCount = getActiveConnections(regionServer, conf); admin.unassign(regionInfo.getEncodedNameAsBytes(), true); getUtility().waitUntilAllRegionsAssigned(TableName.valueOf(tableName)); int afterRegionCloseCount = getActiveConnections(regionServer, conf); assertTrue("Cached connections not closed when region closes: ", afterRegionCloseCount == beforeRegionCloseCount && afterRegionCloseCount > 0); }
private static void getRegionLocations(HBaseAdmin admin, String tableName) throws IOException { try (HTable table = new HTable(admin.getConfiguration(), tableName)) { regionLocations.putAll(table.getRegionLocations()); cachedTableNames.add(tableName); } }
private void clean(NavigableMap<HRegionInfo, ServerName> regionServerMap) throws InterruptedException, IOException { long timestamp = System.currentTimeMillis(); Set<String> tableNameSet = tableNameSet(regionServerMap); ExecutorService executorService = Executors.newFixedThreadPool(RegionLocationCleaner.THREAD_POOL_SIZE); try { for (String tableName : tableNameSet) executorService.execute( new RegionLocationCleaner(tableName, admin.getConfiguration(), regionServerMap)); } finally { executorService.shutdown(); executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); } Util.printVerboseMessage(args, "TableInfo.clean", timestamp); }
private NavigableMap<HRegionInfo, ServerName> getRegionLocations(String tableName, HBaseAdmin admin) throws IOException { try (HTable table = new HTable(admin.getConfiguration(), tableName)) { return table.getRegionLocations(); } } };
public static void waitUntilAssigned(HBaseAdmin admin, HRegionInfo region) throws IOException, InterruptedException { long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000); long expiration = timeout + System.currentTimeMillis(); while (System.currentTimeMillis() < expiration) { try { Map<String, RegionState> rits= admin.getClusterStatus().getRegionsInTransition(); if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) { // yay! no longer RIT return; } // still in rit LOG.info("Region still in transition, waiting for " + "it to become assigned: " + region); } catch (IOException e) { LOG.warn("Exception when waiting for region to become assigned," + " retrying", e); } Thread.sleep(1000); } throw new IOException("Region " + region + " failed to move out of " + "transition within timeout " + timeout + "ms"); }
private static List<RegionPlan> makeStochasticPlan( HBaseAdmin admin, Set<String> tableNameSet, BalanceFactor balanceFactor) throws IOException { Map<ServerName, List<HRegionInfo>> clusterState = CommandAdapter.initializeRegionMap(admin); Map<HRegionInfo, ServerName> regionAssignmentMap = Balance.getRegionAssignmentMap(admin, tableNameSet); for (Map.Entry<HRegionInfo, ServerName> entry : regionAssignmentMap.entrySet()) { List<HRegionInfo> hRegionInfos = clusterState.get(entry.getValue()); if (hRegionInfos != null) { hRegionInfos.add(entry.getKey()); } } Configuration conf = admin.getConfiguration(); conf.setFloat("hbase.regions.slop", 0f); balanceFactor.setConf(conf); return CommandAdapter.makePlan(admin, clusterState, conf); }
public static boolean isMoved(HBaseAdmin admin, String tableName, String regionName, String serverNameTarget) { try (HTable table = new HTable(admin.getConfiguration(), tableName)) { NavigableMap<HRegionInfo, ServerName> regionLocations = table.getRegionLocations(); for (Map.Entry<HRegionInfo, ServerName> regionLocation : regionLocations.entrySet()) { if (regionLocation.getKey().getEncodedName().equals(regionName)) { return regionLocation.getValue().getServerName().equals(serverNameTarget); } } if (!existsRegion(regionName, regionLocations.keySet())) return true; // skip moving } catch (IOException e) { return false; } return false; }
public static List<RegionPlan> makePlan(HBaseAdmin admin, List<RegionPlan> newRegionPlan) throws IOException { // snapshot current region assignment Map<HRegionInfo, ServerName> regionAssignmentMap = createRegionAssignmentMap(admin); // update with new plan for (RegionPlan regionPlan : newRegionPlan) { regionAssignmentMap.put(regionPlan.getRegionInfo(), regionPlan.getDestination()); } Map<ServerName, List<HRegionInfo>> clusterState = initializeRegionMap(admin); for (Map.Entry<HRegionInfo, ServerName> entry : regionAssignmentMap.entrySet()) clusterState.get(entry.getValue()).add(entry.getKey()); StochasticLoadBalancer balancer = new StochasticLoadBalancer(); Configuration conf = admin.getConfiguration(); conf.setFloat("hbase.regions.slop", 0.2f); balancer.setConf(conf); return balancer.balanceCluster(clusterState); }
private NavigableMap<HRegionInfo, ServerName> getRegionLocations(String table) throws IOException { long startTimestamp = System.currentTimeMillis(); Util.printVerboseMessage(args, Util.getMethodName() + " - start"); NavigableMap<HRegionInfo, ServerName> result = regionLocations.get(table); if (result == null) { try (HTable htable = new HTable(admin.getConfiguration(), table)) { result = htable.getRegionLocations(); regionLocations.put(table, result); } } Util.printVerboseMessage(args, Util.getMethodName() + " - end", startTimestamp); return result; }
public Merge(HBaseAdmin admin, Args args) throws IOException { if (args.getOptionSet().nonOptionArguments().size() < 2 || args.getOptionSet().nonOptionArguments().size() > 3) { // todo refactoring throw new RuntimeException(Args.INVALID_ARGUMENTS); } this.admin = admin; this.args = args; if (args.has(Args.OPTION_TEST)) this.test = true; actionParam = (String) args.getOptionSet().nonOptionArguments().get(2); this.connection = HConnectionManager.createConnection(admin.getConfiguration()); tableNameSet = Util.parseTableSet(admin, args); if (args.has(Args.OPTION_PHOENIX)) this.isPhoenixSaltingTable = true; }
private void initializeRegionServerMap() throws Exception { long timestamp = System.currentTimeMillis(); if (load.getLevelClass().getLevelClass() == RegionName.class || args.has(Args.OPTION_REGION_SERVER)) { initializeServerNameSet(); } Set<String> tables = Args.tables(args, admin); if (tables == null) { regionServerMap = CommandAdapter.regionServerMap(args, admin.getConfiguration() , admin.getConnection(), false); } else { regionServerMap = CommandAdapter.regionServerMap(args, admin.getConfiguration() , admin.getConnection(), tables, false); } clean(regionServerMap); Util.printVerboseMessage(args, "TableInfo.initializeRegionServerMap", timestamp); }
private void sniff(HTableDescriptor tableDesc) throws Exception { HTable table = null; try { table = new HTable(admin.getConfiguration(), tableDesc.getName()); } catch (TableNotFoundException e) { return; } for (HRegionInfo region : admin.getTableRegions(tableDesc.getName())) { try { sniffRegion(region, table); } catch (Exception e) { sink.publishReadFailure(region); } } }
private void create(TableId tableId) throws IOException { HBaseTableUtil tableUtil = getTableUtil(); TableId htableId = tableUtil.createHTableId(new NamespaceId(tableId.getNamespace()), tableId.getTableName()); ColumnFamilyDescriptorBuilder cfdBuilder = HBaseTableUtil.getColumnFamilyDescriptorBuilder("d", hAdmin.getConfiguration()); TableDescriptorBuilder tdBuilder = HBaseTableUtil.getTableDescriptorBuilder(htableId, cConf); tdBuilder.addColumnFamily(cfdBuilder.build()); ddlExecutor.createTableIfNotExists(tdBuilder.build(), null); }
@Test public void testDropAllInDefaultNamespace() throws Exception { HBaseTableUtil tableUtil = getTableUtil(); TableId tableIdInOtherNamespace = TableId.from("default2", "my.dataset"); createNamespace("default2"); Futures.allAsList( createAsync(TableId.from("default", "some.table1")), createAsync(TableId.from("default", "other.table")), createAsync(TableId.from("default", "some.table2")), createAsync(tableIdInOtherNamespace) ).get(60, TimeUnit.SECONDS); Assert.assertEquals(4, hAdmin.listTables().length); tableUtil.deleteAllInNamespace(ddlExecutor, NamespaceId.DEFAULT.getEntityName(), hAdmin.getConfiguration()); Assert.assertEquals(1, hAdmin.listTables().length); drop(tableIdInOtherNamespace); Assert.assertEquals(0, hAdmin.listTables().length); deleteNamespace("default2"); }
@Test public void testCustomNamespaceMap() throws Exception { final String tableName = "mytable"; Assert.assertEquals(HBASE_NS, getTableUtil().getHBaseNamespace(new NamespaceId(CDAP_NS))); Assert.assertFalse(getTableUtil().hasNamespace(hAdmin, HBASE_NS)); createNamespace(CDAP_NS); // Check if namespace was created - in actual run, custom namespaces' lifecycle is managed by the user and hence // we won't create the namespace Assert.assertTrue(getTableUtil().hasNamespace(hAdmin, HBASE_NS)); TableId tableId = TableId.from(CDAP_NS, tableName); create(tableId); List<TableId> actualTableIds = getTableUtil().listTablesInNamespace(hAdmin, HBASE_NS); Assert.assertEquals(1, actualTableIds.size()); getTableUtil().deleteAllInNamespace(ddlExecutor, HBASE_NS, hAdmin.getConfiguration()); actualTableIds = getTableUtil().listTablesInNamespace(hAdmin, HBASE_NS); Assert.assertTrue(actualTableIds.isEmpty()); deleteNamespace(CDAP_NS); Assert.assertFalse(getTableUtil().hasNamespace(hAdmin, HBASE_NS)); }