@Override protected Function<TableDescriptorBuilder, TableDescriptorBuilder> getMetaTableObserver() { return builder -> builder.setRegionReplication(conf.getInt(HConstants.META_REPLICAS_NUM, HConstants.DEFAULT_META_REPLICA_NUM)); } /**
private TableDescriptor getTableDesc(TableName tableName, int regionReplication, byte[]... families) { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(regionReplication); Arrays.stream(families).map(family -> ColumnFamilyDescriptorBuilder.newBuilder(family) .setMaxVersions(Integer.MAX_VALUE).build()).forEachOrdered(builder::setColumnFamily); return builder.build(); }
private static Table createTableDirectlyFromHTD(final TableName tableName) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName); builder.setRegionReplication(3); return HTU.createTable(builder.build(), new byte[][] { f }, getSplits(20), new Configuration(HTU.getConfiguration())); }
private static void createTable(final TableName tableName) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName); builder.setRegionReplication(3); // create a table with 3 replication table = HTU.createTable(builder.build(), new byte[][] { f }, getSplits(2), new Configuration(HTU.getConfiguration())); }
@Test(expected = DoNotRetryIOException.class) public void testCreateTableWithZeroRegionReplicas() throws Exception { TableName tableName = TableName.valueOf(name.getMethodName()); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf"))) .setRegionReplication(0) .build(); TEST_UTIL.getAdmin().createTable(desc); }
protected static void startClusterAndCreateTable() throws Exception { // 10 mins TEST_UTIL.getConfiguration().setLong(HConstants.HBASE_RPC_READ_TIMEOUT_KEY, TimeUnit.MINUTES.toMillis(10)); TEST_UTIL.getConfiguration().setLong(HConstants.HBASE_RPC_TIMEOUT_KEY, TimeUnit.MINUTES.toMillis(10)); TEST_UTIL.getConfiguration().setLong(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, TimeUnit.MINUTES.toMillis(10)); // 1 second TEST_UTIL.getConfiguration().setLong(ConnectionConfiguration.PRIMARY_CALL_TIMEOUT_MICROSECOND, TimeUnit.SECONDS.toMicros(1)); TEST_UTIL.getConfiguration().setLong(ConnectionConfiguration.PRIMARY_SCAN_TIMEOUT_MICROSECOND, TimeUnit.SECONDS.toMicros(1)); // set a small pause so we will retry very quickly TEST_UTIL.getConfiguration().setLong(HConstants.HBASE_CLIENT_PAUSE, 10); // infinite retry TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, Integer.MAX_VALUE); TEST_UTIL.startMiniCluster(3); TEST_UTIL.getAdmin().createTable(TableDescriptorBuilder.newBuilder(TABLE_NAME) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).setRegionReplication(REPLICA_COUNT) .setCoprocessor(FailPrimaryGetCP.class.getName()).build()); TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME); ASYNC_CONN = ConnectionFactory.createAsyncConnection(TEST_UTIL.getConfiguration()).get(); }
@Test public void testTruncatePreserveWithReplicaRegionAfterSplit() throws Exception { String[] families = new String[] { "f1", "f2" }; byte[][] splitKeys = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") }; TableName tableName = TableName.valueOf(name.getMethodName()); // create a table with region replications TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(3) .setColumnFamilies(Arrays.stream(families) .map(fam -> ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(fam)).build()) .collect(Collectors.toList())) .build(); RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); long procId = ProcedureTestingUtility.submitAndWait(procExec, new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId)); splitAndTruncate(tableName, regions, 3); }
.setRegionReplication(regionReplication) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); TEST_UTIL.getAdmin().createTable(builder.build());
@Test(expected = DoNotRetryIOException.class) public void testModifyTableWithZeroRegionReplicas() throws Exception { TableName tableName = TableName.valueOf(name.getMethodName()); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf"))) .build(); TEST_UTIL.getAdmin().createTable(desc); TableDescriptor newDesc = TableDescriptorBuilder.newBuilder(desc) .setRegionReplication(0) .build(); TEST_UTIL.getAdmin().modifyTable(newDesc); } }
public static void createTable(final HBaseTestingUtility util, final TableName tableName, int regionReplication, int nRegions, final byte[]... families) throws IOException, InterruptedException { TableDescriptorBuilder builder = TableDescriptorBuilder .newBuilder(tableName) .setRegionReplication(regionReplication); for (byte[] family : families) { builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)); } byte[][] splitKeys = getSplitKeys(nRegions); util.createTable(builder.build(), splitKeys); assertEquals((splitKeys.length + 1) * regionReplication, util.getAdmin().getTableRegions(tableName).size()); }
protected static void startClusterAndCreateTable() throws Exception { UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, REGION_REPLICATION); UTIL.startMiniCluster(3); TableDescriptor td = TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(REGION_REPLICATION) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build(); SPLIT_KEYS = new byte[9][]; for (int i = 0; i < 9; i++) { SPLIT_KEYS[i] = Bytes.toBytes(Integer.toString(i + 1)); } UTIL.getAdmin().createTable(td, SPLIT_KEYS); UTIL.waitTableAvailable(TABLE_NAME); try (AsyncRegistry registry = AsyncRegistryFactory.getRegistry(UTIL.getConfiguration())) { RegionReplicaTestHelper.waitUntilAllMetaReplicasHavingRegionLocation(registry, REGION_REPLICATION); } UTIL.getAdmin().balancerSwitch(false, true); }
private void testRollbackAndDoubleExecution(TableDescriptorBuilder builder) throws Exception { // create the table final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); // Start the Create procedure && kill the executor final byte[][] splitKeys = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c") }; builder.setRegionReplication(3); TableDescriptor htd = builder.build(); RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys); long procId = procExec.submitProcedure( new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); int lastStep = 2; // failing before CREATE_TABLE_WRITE_FS_LAYOUT MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep); TableName tableName = htd.getTableName(); MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName); // are we able to create the table after a rollback? resetProcExecutorTestingKillFlag(); testSimpleCreate(tableName, splitKeys); }
public static void createMobTable(HBaseTestingUtility util, TableName tableName, byte[][] splitKeys, int regionReplication, String cpClassName, byte[]... families) throws IOException, InterruptedException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(regionReplication); for (byte[] family : families) { builder.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setMobEnabled(true) .setMobThreshold(0L).build()); } if (!StringUtils.isBlank(cpClassName)) { builder.setCoprocessor(cpClassName); } util.getAdmin().createTable(builder.build(), splitKeys); SnapshotTestingUtils.waitForTableToBeOnline(util, tableName); assertEquals((splitKeys.length + 1) * regionReplication, util.getAdmin().getRegions(tableName).size()); }
@Test public void testModifyTableOnTableWithRegionReplicas() throws Exception { TableName tableName = TableName.valueOf(name.getMethodName()); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf"))) .setRegionReplication(5) .build(); admin.createTable(desc); int maxFileSize = 10000000; TableDescriptor newDesc = TableDescriptorBuilder.newBuilder(desc) .setMaxFileSize(maxFileSize) .build(); admin.modifyTable(newDesc); TableDescriptor newTableDesc = admin.getDescriptor(tableName); assertEquals(maxFileSize, newTableDesc.getMaxFileSize()); } }
@BeforeClass public static void setUp() throws Exception { UTIL.startMiniCluster(1); UTIL.getAdmin().balancerSwitch(false, true); UTIL.createTable(TableDescriptorBuilder.newBuilder(TABLE_NAME) .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf")) .setRegionReplication(REGION_REPLICATION).build(), new byte[][] { Bytes.toBytes(0) }); UTIL.waitTableAvailable(TABLE_NAME); HMaster master = UTIL.getMiniHBaseCluster().getMaster(); ENV = master.getMasterProcedureExecutor().getEnvironment(); AM = master.getAssignmentManager(); }
@Test(expected = TableNotDisabledException.class) public void testModifyRegionReplicasEnabledTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); // Modify region replication count TableDescriptor htd = TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName)) .setRegionReplication(3).build(); try { // try to modify the region replication count without disabling the table admin.modifyTable(htd); fail("Expected an exception"); } finally { // Delete the table admin.disableTable(tableName); admin.deleteTable(tableName); assertFalse(admin.tableExists(tableName)); } }
@Test public void testRegionReplicas() throws Exception { TEST_UTIL.getAdmin().createTable(TableDescriptorBuilder.newBuilder(TABLE_NAME) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).setRegionReplication(3).build()); TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME); testLocator(TEST_UTIL, TABLE_NAME, new Locator() { @Override public void updateCachedLocationOnError(HRegionLocation loc, Throwable error) throws Exception { LOCATOR.updateCachedLocationOnError(loc, error); } @Override public RegionLocations getRegionLocations(TableName tableName, int replicaId, boolean reload) throws Exception { return LOCATOR.getRegionLocations(tableName, EMPTY_START_ROW, replicaId, RegionLocateType.CURRENT, reload).get(); } }); } }
@Test public void testEnableReplicationForTableWithRegionReplica() throws Exception { TableName tn = TableName.valueOf(name.getMethodName()); TableDescriptor td = TableDescriptorBuilder.newBuilder(tn) .setRegionReplication(5) .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(noRepfamName).build()) .build(); admin1.createTable(td); try { admin1.enableTableReplication(tn); td = admin1.getDescriptor(tn); for (ColumnFamilyDescriptor fam : td.getColumnFamilies()) { assertEquals(HConstants.REPLICATION_SCOPE_GLOBAL, fam.getScope()); } } finally { utility1.deleteTable(tn); utility2.deleteTable(tn); } }
@Test public void testPb() throws DeserializationException, IOException { final int v = 123; TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.META_TABLE_NAME) .setMaxFileSize(v) .setDurability(Durability.ASYNC_WAL) .setReadOnly(true) .setRegionReplication(2) .build(); byte [] bytes = TableDescriptorBuilder.toByteArray(htd); TableDescriptor deserializedHtd = TableDescriptorBuilder.parseFrom(bytes); assertEquals(htd, deserializedHtd); assertEquals(v, deserializedHtd.getMaxFileSize()); assertTrue(deserializedHtd.isReadOnly()); assertEquals(Durability.ASYNC_WAL, deserializedHtd.getDurability()); assertEquals(2, deserializedHtd.getRegionReplication()); }
TableDescriptor newTd = TableDescriptorBuilder.newBuilder(td).setCompactionEnabled(!td.isCompactionEnabled()) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(familyName)).setRegionReplication(3) .build();