/** * Converts a TableSchema to TableDescriptor * @param ts A pb TableSchema instance. * @return An {@link TableDescriptor} made from the passed in pb <code>ts</code>. */ public static TableDescriptor toTableDescriptor(final TableSchema ts) { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(ProtobufUtil.toTableName(ts.getTableName())); ts.getColumnFamiliesList() .stream() .map(ProtobufUtil::toColumnFamilyDescriptor) .forEach(builder::setColumnFamily); ts.getAttributesList() .forEach(a -> builder.setValue(a.getFirst().toByteArray(), a.getSecond().toByteArray())); ts.getConfigurationList() .forEach(a -> builder.setValue(a.getName(), a.getValue())); return builder.build(); }
public static TableDescriptor tableDescriptorFromThrift(TTableDescriptor in) { TableDescriptorBuilder builder = TableDescriptorBuilder .newBuilder(tableNameFromThrift(in.getTableName())); for (TColumnFamilyDescriptor column : in.getColumns()) { builder.setColumnFamily(columnFamilyDescriptorFromThrift(column)); } if (in.isSetAttributes()) { for (Map.Entry<ByteBuffer, ByteBuffer> attribute : in.getAttributes().entrySet()) { builder.setValue(attribute.getKey().array(), attribute.getValue().array()); } } if (in.isSetDurability()) { builder.setDurability(durabilityFromThrift(in.getDurability())); } return builder.build(); }
@Before public void setUp() throws Exception { MemoryCompactionPolicy policy = getMemoryCompactionPolicy(); TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TABLE_NAME) .setValue(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, policy.name()); if (policy == MemoryCompactionPolicy.EAGER) { builder.setValue(MemStoreLAB.USEMSLAB_KEY, "false"); builder.setValue(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, "0.9"); } Stream.of(FAMILIES).map(ColumnFamilyDescriptorBuilder::of) .forEachOrdered(builder::setColumnFamily); UTIL.getAdmin().createTable(builder.build()); tool.setConf(UTIL.getConfiguration()); }
@Test public void testSanityCheckTTL() throws IOException { error.expect(DoNotRetryIOException.class); error.expectMessage("Default TTL is not supported"); TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-TTL"); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build(); TEST_UTIL.getAdmin().createTable(desc); }
@BeforeClass public static void setUp() throws Exception { UTIL.startMiniCluster(1); UTIL.getAdmin() .createTable(TableDescriptorBuilder.newBuilder(NAME) .setCoprocessor(ScanModifyingObserver.class.getName()) .setValue(ScanModifyingObserver.FAMILY_TO_ADD_KEY, Bytes.toString(FAMILY)) .setValue(ScanModifyingObserver.QUALIFIER_TO_ADD_KEY, Bytes.toString(IMPLICIT_QUAL)) .setColumnFamily(CFD).build()); }
@Test public void testSanityCheckBlockingStoreFiles() throws IOException { error.expect(DoNotRetryIOException.class); error.expectMessage("Blocking file count 'hbase.hstore.blockingStoreFiles'"); error.expectMessage("is below recommended minimum of 1000 for column family"); TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-BlockingStoreFiles"); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()) .setValue(HStore.BLOCKING_STOREFILES_KEY, "10") .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build()) .build(); TEST_UTIL.getAdmin().createTable(desc); }
@BeforeClass public static void setUp() throws Exception { UTIL.startMiniCluster(1); UTIL.getAdmin() .createTable(TableDescriptorBuilder.newBuilder(NAME) .setCoprocessor(ValueRewritingObserver.class.getName()) .setValue(ValueRewritingObserver.ORIGINAL_VALUE_KEY, value) .setValue(ValueRewritingObserver.REPLACED_VALUE_KEY, replacedValue) .setColumnFamily(CFD).build()); }
@Test public void testSanityCheckMinVersion() throws IOException { error.expect(DoNotRetryIOException.class); error.expectMessage("MIN_VERSION > 0 is not supported for FIFO compaction"); TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-MinVersion"); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()) .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1) .setMinVersions(1).build()) .build(); TEST_UTIL.getAdmin().createTable(desc); }
private Table setupTable(TableName tableName) throws Exception { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName); // Flush many files, but do not compact immediately // Make sure that the region does not split builder .setMemStoreFlushSize(5000) .setRegionSplitPolicyClassName(ConstantSizeRegionSplitPolicy.class.getName()) .setMaxFileSize(100 * 1024 * 1024) .setValue("hbase.hstore.compactionThreshold", "250"); TableDescriptor td = builder.build(); byte[] fam = Bytes.toBytes("fam"); Table table = TEST_UTIL.createTable(td, new byte[][] {fam}, TEST_UTIL.getConfiguration()); TEST_UTIL.loadTable(table, fam); return table; } }
@Before public void before() throws Exception { TableDescriptor desc = TableDescriptorBuilder.newBuilder(TABLE_NAME) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)) .setValue(HRegion.HBASE_MAX_CELL_SIZE_KEY, String.valueOf(CELL_SIZE)).build(); TEST_UTIL.getConnection().getAdmin().createTable(desc); }
@BeforeClass public static void setUp() throws Exception { UTIL.startMiniCluster(3); UTIL.getAdmin() .createTable(TableDescriptorBuilder.newBuilder(NAME) .setCoprocessor(ZooKeeperScanPolicyObserver.class.getName()) .setValue(ZooKeeperScanPolicyObserver.ZK_ENSEMBLE_KEY, "localhost:" + UTIL.getZkCluster().getClientPort()) .setValue(ZooKeeperScanPolicyObserver.ZK_SESSION_TIMEOUT_KEY, "2000") .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).build()).build()); TABLE = UTIL.getConnection().getTable(NAME); }
private HStore prepareData() throws IOException { Admin admin = TEST_UTIL.getAdmin(); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()) .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build()) .build(); admin.createTable(desc); Table table = TEST_UTIL.getConnection().getTable(tableName); TimeOffsetEnvironmentEdge edge = (TimeOffsetEnvironmentEdge) EnvironmentEdgeManager.getDelegate(); for (int i = 0; i < 10; i++) { for (int j = 0; j < 10; j++) { byte[] value = new byte[128 * 1024]; ThreadLocalRandom.current().nextBytes(value); table.put(new Put(Bytes.toBytes(i * 10 + j)).addColumn(family, qualifier, value)); } admin.flush(tableName); edge.increment(1001); } return getStoreWithName(tableName); }
/** * Test that we add and remove strings from settings properly. * @throws Exception */ @Test public void testRemoveString() throws Exception { byte[] key = Bytes.toBytes("Some"); byte[] value = Bytes.toBytes("value"); TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())) .setValue(key, value) .build(); assertTrue(Bytes.equals(value, desc.getValue(key))); desc = TableDescriptorBuilder.newBuilder(desc) .removeValue(key) .build(); assertTrue(desc.getValue(key) == null); }
@Test public void testCreateTableDescriptorUpdatesIfExistsAlready() throws IOException { Path testdir = UTIL.getDataTestDir(name.getMethodName()); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName())).build(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, testdir); assertTrue(fstd.createTableDescriptor(htd)); assertFalse(fstd.createTableDescriptor(htd)); htd = TableDescriptorBuilder.newBuilder(htd) .setValue(Bytes.toBytes("mykey"), Bytes.toBytes("myValue")) .build(); assertTrue(fstd.createTableDescriptor(htd)); //this will re-create Path tableDir = fstd.getTableDir(htd.getTableName()); Path tmpTableDir = new Path(tableDir, FSTableDescriptors.TMP_DIR); FileStatus[] statuses = fs.listStatus(tmpTableDir); assertTrue(statuses.length == 0); assertEquals(htd, FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir)); }
@Test public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception { final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle"; long anyValue = 10; // We'll check that it uses correct config and propagates it appropriately by going thru // the simplest "real" path I can find - "throttleCompaction", which just checks whether // a number we pass in is higher than some config value, inside compactionPolicy. Configuration conf = HBaseConfiguration.create(); conf.setLong(CONFIG_KEY, anyValue); init(name.getMethodName() + "-xml", conf); assertTrue(store.throttleCompaction(anyValue + 1)); assertFalse(store.throttleCompaction(anyValue)); // HTD overrides XML. --anyValue; init(name.getMethodName() + "-htd", conf, TableDescriptorBuilder .newBuilder(TableName.valueOf(table)).setValue(CONFIG_KEY, Long.toString(anyValue)), ColumnFamilyDescriptorBuilder.of(family)); assertTrue(store.throttleCompaction(anyValue + 1)); assertFalse(store.throttleCompaction(anyValue)); // HCD overrides them both. --anyValue; init(name.getMethodName() + "-hcd", conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setValue(CONFIG_KEY, Long.toString(anyValue)), ColumnFamilyDescriptorBuilder.newBuilder(family).setValue(CONFIG_KEY, Long.toString(anyValue)) .build()); assertTrue(store.throttleCompaction(anyValue + 1)); assertFalse(store.throttleCompaction(anyValue)); }
@BeforeClass public static void setUp() throws Exception { WriteHeavyIncrementObserverTestBase.setUp(); UTIL.getAdmin() .createTable(TableDescriptorBuilder.newBuilder(NAME) .setCoprocessor(WriteHeavyIncrementObserver.class.getName()) .setValue(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, MemoryCompactionPolicy.EAGER.name()) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build()); TABLE = UTIL.getConnection().getTable(NAME); }
TableName tableName = TableName.valueOf("testFIFOCompactionPolicyExpiredEmptyHFiles"); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()) .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1).build())
@Test public void testCreateWithoutColumnFamily() throws Exception { final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor(); final TableName tableName = TableName.valueOf(name.getMethodName()); // create table with 0 families will fail final TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(MasterProcedureTestingUtility.createHTD(tableName)); // disable sanity check builder.setValue("hbase.table.sanity.checks", Boolean.FALSE.toString()); TableDescriptor htd = builder.build(); final RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, null); long procId = ProcedureTestingUtility.submitAndWait(procExec, new CreateTableProcedure(procExec.getEnvironment(), htd, regions)); final Procedure<?> result = procExec.getResult(procId); assertEquals(true, result.isFailed()); Throwable cause = ProcedureTestingUtility.getExceptionCause(result); assertTrue("expected DoNotRetryIOException, got " + cause, cause instanceof DoNotRetryIOException); }
private void setTransactional(byte[] physicalTableName, TableDescriptorBuilder tableDescriptorBuilder, PTableType tableType, String txValue, Map<String, Object> tableProps) throws SQLException { if (txValue == null) { tableDescriptorBuilder.removeValue(Bytes.toBytes(PhoenixTransactionContext.READ_NON_TX_DATA)); } else { tableDescriptorBuilder.setValue(PhoenixTransactionContext.READ_NON_TX_DATA, txValue); } this.addCoprocessors(physicalTableName, tableDescriptorBuilder, tableType, tableProps); }
cfd.setMaxVersions(100); tableBuilder.setColumnFamily(cfd.build()); tableBuilder.setValue(TableDescriptorBuilder.OWNER, USER_OWNER.getShortName()); createTable(TEST_UTIL, tableBuilder.build(), new byte[][] { Bytes.toBytes("s") });