private HStore init(String methodName, Configuration conf) throws IOException { // some of the tests write 4 versions and then flush // (with HBASE-4241, lower versions are collected on flush) return init(methodName, conf, ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(4).build()); }
@Before public void setUp() throws Exception { baseConfiguration = HBaseConfiguration.create(); // smaller block size and capacity to trigger more operations // and test them baseConfiguration.setInt("hbase.regionserver.hlog.blocksize", 1024 * 20); baseConfiguration.setInt("replication.source.size.capacity", 1024); baseConfiguration.setLong("replication.source.sleepforretries", 100); baseConfiguration.setInt("hbase.regionserver.maxlogs", 10); baseConfiguration.setLong("hbase.master.logcleaner.ttl", 10); baseConfiguration.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true); baseConfiguration.set("hbase.replication.source.fs.conf.provider", TestSourceFSConfigurationProvider.class.getCanonicalName()); baseConfiguration.set(HConstants.REPLICATION_CLUSTER_ID, "12345"); baseConfiguration.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100); baseConfiguration.setStrings( CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY, CoprocessorCounter.class.getName()); table = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName) .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()) .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName1) .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(noRepfamName)).build(); }
public static ColumnFamilyDescriptor columnFamilyDescriptorFromThrift( TColumnFamilyDescriptor in) { ColumnFamilyDescriptorBuilder builder = ColumnFamilyDescriptorBuilder .newBuilder(in.getName()); builder.setValue(attribute.getKey().array(), attribute.getValue().array()); builder.setConfiguration(conf.getKey(), conf.getValue()); builder.setBlocksize(in.getBlockSize()); builder.setBloomFilterType(bloomFilterFromThrift(in.getBloomnFilterType())); builder.setCompressionType(compressionAlgorithmFromThrift(in.getCompressionType())); builder.setDFSReplication(in.getDfsReplication()); builder.setDataBlockEncoding(dataBlockEncodingFromThrift(in.getDataBlockEncoding())); builder.setKeepDeletedCells(keepDeletedCellsFromThrift(in.getKeepDeletedCells())); builder.setMaxVersions(in.getMaxVersions()); builder.setMinVersions(in.getMinVersions()); builder.setScope(in.getScope()); builder.setTimeToLive(in.getTimeToLive()); builder.setBlockCacheEnabled(in.isBlockCacheEnabled());
/** * Create the ACL table * @throws IOException */ private static void createACLTable(Admin admin) throws IOException { /** Table descriptor for ACL table */ ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(AccessControlLists.ACL_LIST_FAMILY). setMaxVersions(1). setInMemory(true). setBlockCacheEnabled(true). setBlocksize(8 * 1024). setBloomFilterType(BloomType.NONE). setScope(HConstants.REPLICATION_SCOPE_LOCAL).build(); TableDescriptor td = TableDescriptorBuilder.newBuilder(AccessControlLists.ACL_TABLE_NAME). setColumnFamily(cfd).build(); admin.createTable(td); }
@Override protected ColumnFamilyDescriptor getTestRestoreSchemaChangeHCD() { return ColumnFamilyDescriptorBuilder.newBuilder(TEST_FAMILY2).setMobEnabled(true) .setMobThreshold(3L).build(); } }
/** * Sets replication scope all & only the columns already in the builder. Columns added later won't * be backfilled with replication scope. * @param scope replication scope * @return a TableDescriptorBuilder */ public TableDescriptorBuilder setReplicationScope(int scope) { Map<byte[], ColumnFamilyDescriptor> newFamilies = new TreeMap<>(Bytes.BYTES_RAWCOMPARATOR); newFamilies.putAll(desc.families); newFamilies .forEach((cf, cfDesc) -> { desc.removeColumnFamily(cf); desc.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(cfDesc).setScope(scope) .build()); }); return this; }
@Test public void testBuilder() throws DeserializationException { ColumnFamilyDescriptorBuilder builder = ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY) .setInMemory(true) .setScope(HConstants.REPLICATION_SCOPE_LOCAL) .setBloomFilterType(BloomType.NONE); final int v = 123; builder.setBlocksize(v); builder.setTimeToLive(v); builder.setBlockCacheEnabled(!HColumnDescriptor.DEFAULT_BLOCKCACHE); builder.setValue(Bytes.toBytes("a"), Bytes.toBytes("b")); builder.setMaxVersions(v); assertEquals(v, builder.build().getMaxVersions()); builder.setMinVersions(v); assertEquals(v, builder.build().getMinVersions()); builder.setKeepDeletedCells(KeepDeletedCells.TRUE); builder.setInMemory(!HColumnDescriptor.DEFAULT_IN_MEMORY); boolean inmemory = builder.build().isInMemory(); builder.setScope(v); builder.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF); builder.setBloomFilterType(BloomType.ROW); builder.setCompressionType(Algorithm.SNAPPY); builder.setMobEnabled(true); builder.setMobThreshold(1000L); builder.setDFSReplication((short) v); ColumnFamilyDescriptor hcd = builder.build(); byte [] bytes = ColumnFamilyDescriptorBuilder.toByteArray(hcd); ColumnFamilyDescriptor deserializedHcd = ColumnFamilyDescriptorBuilder.parseFrom(bytes);
String ttl; ColumnFamilyDescriptorBuilder builder = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("foo")); builder.setTimeToLive(ttl); Assert.assertEquals(50000, builder.build().getTimeToLive()); builder.setTimeToLive(ttl); Assert.assertEquals(50000, builder.build().getTimeToLive()); builder.setTimeToLive(ttl); Assert.assertEquals(0, builder.build().getTimeToLive()); builder.setTimeToLive(ttl); Assert.assertEquals(HConstants.FOREVER, builder.build().getTimeToLive()); builder.setTimeToLive(ttl); Assert.assertEquals(4201, builder.build().getTimeToLive()); builder.setTimeToLive(ttl); Assert.assertEquals(43282800, builder.build().getTimeToLive()); builder.setTimeToLive(ttl); Assert.assertEquals(43282800, builder.build().getTimeToLive());
final int maxVersions = 3; ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(cfBytes).setCompressionType(compress) .setBloomFilterType(BLOOM_TYPE).setMaxVersions(maxVersions) .setDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding()).build(); HRegion region = TEST_UTIL.createTestRegion(table, cfd, blockCache); int rowIdx = 0;
boolean isNewCf = columnDesc == null; ColumnFamilyDescriptorBuilder columnDescBuilder = isNewCf ? ColumnFamilyDescriptorBuilder.newBuilder(cf) : ColumnFamilyDescriptorBuilder.newBuilder(columnDesc); if (bloomType != null) { columnDescBuilder.setBloomFilterType(bloomType); columnDescBuilder.setCompressionType(compressAlgo); columnDescBuilder.setDataBlockEncoding(dataBlockEncodingAlgo); columnDescBuilder.setInMemory(inMemoryCF); columnDescBuilder.setEncryptionType(cipher.getName()); columnDescBuilder.setEncryptionKey( EncryptionUtil.wrapKey(conf, User.getCurrent().getShortName(), columnDescBuilder.setMobEnabled(true); columnDescBuilder.setMobThreshold(mobThreshold); admin.addColumnFamily(tableName, columnDescBuilder.build()); } else { admin.modifyColumnFamily(tableName, columnDescBuilder.build());
.setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY_0)) .setColumnFamily(ColumnFamilyDescriptorBuilder .newBuilder(FAMILY_1) .setBlocksize(BLOCK_SIZE) .setBlockCacheEnabled(BLOCK_CACHE) .setTimeToLive(TTL) .build()).build(); admin.createTable(tableDesc, splitKeys).join();
/** * Get backup system table descriptor * @return table's descriptor */ public static TableDescriptor getSystemTableDescriptor(Configuration conf) { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(getTableName(conf)); ColumnFamilyDescriptorBuilder colBuilder = ColumnFamilyDescriptorBuilder.newBuilder(SESSIONS_FAMILY); colBuilder.setMaxVersions(1); Configuration config = HBaseConfiguration.create(); int ttl = config.getInt(BackupRestoreConstants.BACKUP_SYSTEM_TTL_KEY, BackupRestoreConstants.BACKUP_SYSTEM_TTL_DEFAULT); colBuilder.setTimeToLive(ttl); ColumnFamilyDescriptor colSessionsDesc = colBuilder.build(); builder.setColumnFamily(colSessionsDesc); colBuilder = ColumnFamilyDescriptorBuilder.newBuilder(META_FAMILY); colBuilder.setTimeToLive(ttl); builder.setColumnFamily(colBuilder.build()); return builder.build(); }
private static Table createTable() throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TABLE_NAME); builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF1)); builder.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(CF2).setTimeToLive(TTL) .build()); admin.createTable(builder.build()); return TEST_UTIL.getConnection().getTable(TABLE_NAME); }
ColumnFamilyDescriptorBuilder.newBuilder(CF_BYTES).setMaxVersions(MAX_VERSIONS). setDataBlockEncoding(encoding). setBlocksize(BLOCK_SIZE). setBloomFilterType(BloomType.NONE). setCompressTags(compressTags).build(); HRegion region = testUtil.createTestRegion(TABLE_NAME, cfd, cache);
@Test public void testCacheBlocks() throws IOException { // Set index block size to be the same as normal block size. TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, BLOCK_SIZE); BlockCache blockCache = BlockCacheFactory.createBlockCache(TEST_UTIL.getConfiguration()); ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CF)).setMaxVersions(MAX_VERSIONS) .setCompressionType(COMPRESSION_ALGORITHM).setBloomFilterType(BLOOM_TYPE) .setBlocksize(BLOCK_SIZE).setBlockCacheEnabled(cfCacheEnabled).build(); HRegion region = TEST_UTIL.createTestRegion(TABLE, cfd, blockCache); CacheStats stats = blockCache.getStats(); writeTestData(region); assertEquals(0, stats.getHitCount()); assertEquals(0, HFile.DATABLOCK_READ_COUNT.sum()); // Do a single get, take count of caches. If we are NOT caching DATA blocks, the miss // count should go up. Otherwise, all should be cached and the miss count should not rise. region.get(new Get(Bytes.toBytes("row" + 0))); assertTrue(stats.getHitCount() > 0); assertTrue(HFile.DATABLOCK_READ_COUNT.sum() > 0); long missCount = stats.getMissCount(); region.get(new Get(Bytes.toBytes("row" + 0))); if (this.cfCacheEnabled) assertEquals(missCount, stats.getMissCount()); else assertTrue(stats.getMissCount() > missCount); }
private TableDescriptor buildHTD(TableName tableName, BloomType bloomType) { return TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily( ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).setBloomFilterType(bloomType).build()) .build(); }
private ColumnFamilyDescriptor createFamilyDesc() { String familyName = String.format("cf-%010d", RandomUtils.nextInt()); return ColumnFamilyDescriptorBuilder.of(familyName); } }
/** * Callers must afterward call {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} */ private HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf, String family, BlockCache blockCache) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)); for (int i = 0; i < BLOOM_TYPE.length; i++) { BloomType bloomType = BLOOM_TYPE[i]; builder.setColumnFamily( ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(family + "_" + bloomType)) .setBlocksize(1).setBloomFilterType(bloomType).build()); } RegionInfo info = RegionInfoBuilder.newBuilder(TableName.valueOf(tableName)).build(); Path path = new Path(DIR + callingMethod); if (blockCache != null) { return HBaseTestingUtility.createRegionAndWAL(info, path, conf, builder.build(), blockCache); } else { return HBaseTestingUtility.createRegionAndWAL(info, path, conf, builder.build()); } }
@Test public void testSanityCheckMinVersion() throws IOException { error.expect(DoNotRetryIOException.class); error.expectMessage("MIN_VERSION > 0 is not supported for FIFO compaction"); TableName tableName = TableName.valueOf(getClass().getSimpleName() + "-MinVersion"); TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName) .setValue(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY, FIFOCompactionPolicy.class.getName()) .setValue(HConstants.HBASE_REGION_SPLIT_POLICY_KEY, DisabledRegionSplitPolicy.class.getName()) .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(family).setTimeToLive(1) .setMinVersions(1).build()) .build(); TEST_UTIL.getAdmin().createTable(desc); }
private void testSplitStoreFileWithDifferentEncoding(DataBlockEncoding bulkloadEncoding, DataBlockEncoding cfEncoding) throws IOException { Path dir = util.getDataTestDirOnTestFS("testSplitHFileWithDifferentEncoding"); FileSystem fs = util.getTestFileSystem(); Path testIn = new Path(dir, "testhfile"); ColumnFamilyDescriptor familyDesc = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY).setDataBlockEncoding(cfEncoding).build(); HFileTestUtil.createHFileWithDataBlockEncoding(util.getConfiguration(), fs, testIn, bulkloadEncoding, FAMILY, QUALIFIER, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000); Path bottomOut = new Path(dir, "bottom.out"); Path topOut = new Path(dir, "top.out"); LoadIncrementalHFiles.splitStoreFile(util.getConfiguration(), testIn, familyDesc, Bytes.toBytes("ggg"), bottomOut, topOut); int rowCount = verifyHFile(bottomOut); rowCount += verifyHFile(topOut); assertEquals(1000, rowCount); }