@Override public void perform() throws IOException { final int versions = random.nextInt(3) + 1; LOG.debug("Performing action: Changing versions on " + tableName + " to " + versions); modifyAllTableColumns(tableName, columnBuilder -> { columnBuilder.setMinVersions(versions).setMaxVersions(versions); }); LOG.debug("Performing action: Just changed versions on " + tableName); } }
private TableDescriptor getTableDesc(TableName tableName, byte[]... families) { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName); Arrays.stream(families).map( f -> ColumnFamilyDescriptorBuilder.newBuilder(f).setMaxVersions(Integer.MAX_VALUE).build()) .forEachOrdered(builder::setColumnFamily); return builder.build(); }
private TableDescriptor getTableDesc(TableName tableName, int regionReplication, byte[]... families) { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(regionReplication); Arrays.stream(families).map(family -> ColumnFamilyDescriptorBuilder.newBuilder(family) .setMaxVersions(Integer.MAX_VALUE).build()).forEachOrdered(builder::setColumnFamily); return builder.build(); }
private HStore init(String methodName, Configuration conf) throws IOException { // some of the tests write 4 versions and then flush // (with HBASE-4241, lower versions are collected on flush) return init(methodName, conf, ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(4).build()); }
private MyStore initMyStore(String methodName, Configuration conf, MyStoreHook hook) throws IOException { return (MyStore) init(methodName, conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)), ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(5).build(), hook); }
/** * Get backup system table descriptor * @return table's descriptor */ public static TableDescriptor getSystemTableDescriptor(Configuration conf) { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(getTableName(conf)); ColumnFamilyDescriptorBuilder colBuilder = ColumnFamilyDescriptorBuilder.newBuilder(SESSIONS_FAMILY); colBuilder.setMaxVersions(1); Configuration config = HBaseConfiguration.create(); int ttl = config.getInt(BackupRestoreConstants.BACKUP_SYSTEM_TTL_KEY, BackupRestoreConstants.BACKUP_SYSTEM_TTL_DEFAULT); colBuilder.setTimeToLive(ttl); ColumnFamilyDescriptor colSessionsDesc = colBuilder.build(); builder.setColumnFamily(colSessionsDesc); colBuilder = ColumnFamilyDescriptorBuilder.newBuilder(META_FAMILY); colBuilder.setTimeToLive(ttl); builder.setColumnFamily(colBuilder.build()); return builder.build(); }
/** * Get backup system table descriptor * @return table's descriptor */ public static TableDescriptor getSystemTableForBulkLoadedDataDescriptor(Configuration conf) { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(getTableNameForBulkLoadedData(conf)); ColumnFamilyDescriptorBuilder colBuilder = ColumnFamilyDescriptorBuilder.newBuilder(SESSIONS_FAMILY); colBuilder.setMaxVersions(1); Configuration config = HBaseConfiguration.create(); int ttl = config.getInt(BackupRestoreConstants.BACKUP_SYSTEM_TTL_KEY, BackupRestoreConstants.BACKUP_SYSTEM_TTL_DEFAULT); colBuilder.setTimeToLive(ttl); ColumnFamilyDescriptor colSessionsDesc = colBuilder.build(); builder.setColumnFamily(colSessionsDesc); colBuilder = ColumnFamilyDescriptorBuilder.newBuilder(META_FAMILY); colBuilder.setTimeToLive(ttl); builder.setColumnFamily(colBuilder.build()); return builder.build(); }
private void init(String methodName, Configuration conf, boolean testStore) throws IOException { ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(family).setMobEnabled(true).setMobThreshold(3L) .setMaxVersions(4).build(); init(methodName, conf, cfd, testStore); }
@Override public TableDescriptor preModifyTable(ObserverContext<MasterCoprocessorEnvironment> env, TableName tableName, final TableDescriptor currentDescriptor, final TableDescriptor newDescriptor) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(newDescriptor); for (ColumnFamilyDescriptor cfd : newDescriptor.getColumnFamilies()) { builder.modifyColumnFamily( ColumnFamilyDescriptorBuilder.newBuilder(cfd).setMaxVersions(1).build()); } return builder.build(); } }
@Override public TableDescriptor preCreateTableRegionsInfos( ObserverContext<MasterCoprocessorEnvironment> ctx, TableDescriptor desc) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(desc); for (ColumnFamilyDescriptor cfd : desc.getColumnFamilies()) { builder.modifyColumnFamily( ColumnFamilyDescriptorBuilder.newBuilder(cfd).setMaxVersions(1).build()); } return builder.build(); }
private TableDescriptor createTable(Admin admin, String tableName) throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder .newBuilder(TableName.valueOf(tableName)); ColumnFamilyDescriptorBuilder familyABuilder = ColumnFamilyDescriptorBuilder .newBuilder(FAMILYA); familyABuilder.setMaxVersions(3); ColumnFamilyDescriptorBuilder familyBBuilder = ColumnFamilyDescriptorBuilder .newBuilder(FAMILYB); familyBBuilder.setMaxVersions(3); ColumnFamilyDescriptorBuilder familyCBuilder = ColumnFamilyDescriptorBuilder .newBuilder(FAMILYC); familyCBuilder.setMaxVersions(3); builder.setColumnFamily(familyABuilder.build()); builder.setColumnFamily(familyBBuilder.build()); builder.setColumnFamily(familyCBuilder.build()); TableDescriptor tableDescriptor = builder.build(); admin.createTable(tableDescriptor); try (Table table = TEST_UTIL.getConnection().getTable(TableName.valueOf(tableName))) { Put put = new Put(ROW_1); put.addColumn(FAMILYA, QUALIFIER_1, TS_2, VALUE_1); table.put(put); put = new Put(ROW_2); put.addColumn(FAMILYA, QUALIFIER_1, TS_1, VALUE_1); put.addColumn(FAMILYA, QUALIFIER_1, TS_2, VALUE_2); put.addColumn(FAMILYB, QUALIFIER_2, TS_2, VALUE_2); table.put(put); } return tableDescriptor; }
/** * Create the ACL table * @throws IOException */ private static void createACLTable(Admin admin) throws IOException { /** Table descriptor for ACL table */ ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(AccessControlLists.ACL_LIST_FAMILY). setMaxVersions(1). setInMemory(true). setBlockCacheEnabled(true). setBlocksize(8 * 1024). setBloomFilterType(BloomType.NONE). setScope(HConstants.REPLICATION_SCOPE_LOCAL).build(); TableDescriptor td = TableDescriptorBuilder.newBuilder(AccessControlLists.ACL_TABLE_NAME). setColumnFamily(cfd).build(); admin.createTable(td); }
private void createTable(TableName tableName, int maxVersions) throws IOException { ColumnFamilyDescriptorBuilder builder = ColumnFamilyDescriptorBuilder.newBuilder(fam); if (maxVersions > 0) { builder.setMaxVersions(maxVersions); } TEST_UTIL.getAdmin().createTable( TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(builder.build()).build()); }
ColumnFamilyDescriptorBuilder.newBuilder(CF_BYTES).setMaxVersions(MAX_VERSIONS). setDataBlockEncoding(encoding). setBlocksize(BLOCK_SIZE).
@Test public void testMasterObserverToModifyTableSchema() throws IOException { TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TABLENAME); for (int i = 1; i <= 3; i++) { builder.setColumnFamily( ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf" + i)).setMaxVersions(i) .build()); } try (Admin admin = UTIL.getAdmin()) { admin.createTable(builder.build()); assertOneVersion(admin.getDescriptor(TABLENAME)); builder.modifyColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf1")) .setMaxVersions(Integer.MAX_VALUE).build()); admin.modifyTable(builder.build()); assertOneVersion(admin.getDescriptor(TABLENAME)); } }
private void testTimeRangeIfSomeCellsAreDroppedInFlush(int maxVersion) throws IOException { init(this.name.getMethodName(), TEST_UTIL.getConfiguration(), ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(maxVersion).build()); long currentTs = 100; long minTs = currentTs; // the extra cell won't be flushed to disk, // so the min of timerange will be different between memStore and hfile. for (int i = 0; i != (maxVersion + 1); ++i) { this.store.add(new KeyValue(row, family, qf1, ++currentTs, (byte[])null), null); if (i == 1) { minTs = currentTs; } } flushStore(store, id++); Collection<HStoreFile> files = store.getStorefiles(); assertEquals(1, files.size()); HStoreFile f = files.iterator().next(); f.initReader(); StoreFileReader reader = f.getReader(); assertEquals(minTs, reader.timeRange.getMin()); assertEquals(currentTs, reader.timeRange.getMax()); }
/** * Test export scanner batching */ @Test public void testExportScannerBatching() throws Throwable { TableDescriptor desc = TableDescriptorBuilder .newBuilder(TableName.valueOf(name.getMethodName())) .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(FAMILYA) .setMaxVersions(1) .build()) .build(); UTIL.getAdmin().createTable(desc); try (Table t = UTIL.getConnection().getTable(desc.getTableName())) { Put p = new Put(ROW1); p.addColumn(FAMILYA, QUAL, now, QUAL); p.addColumn(FAMILYA, QUAL, now + 1, QUAL); p.addColumn(FAMILYA, QUAL, now + 2, QUAL); p.addColumn(FAMILYA, QUAL, now + 3, QUAL); p.addColumn(FAMILYA, QUAL, now + 4, QUAL); t.put(p); String[] args = new String[] { "-D" + ExportUtils.EXPORT_BATCHING + "=" + EXPORT_BATCH_SIZE, // added scanner batching arg. name.getMethodName(), FQ_OUTPUT_DIR }; assertTrue(runExport(args)); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); fs.delete(new Path(FQ_OUTPUT_DIR), true); } }
private static void setupTables() throws IOException { // Get the table name. tableName = TableName.valueOf(util.getConfiguration() .get("hbase.IntegrationTestMTTR.tableName", "IntegrationTestMTTR")); loadTableName = TableName.valueOf(util.getConfiguration() .get("hbase.IntegrationTestMTTR.loadTableName", "IntegrationTestMTTRLoadTestTool")); if (util.getAdmin().tableExists(tableName)) { util.deleteTable(tableName); } if (util.getAdmin().tableExists(loadTableName)) { util.deleteTable(loadTableName); } // Create the table. If this fails then fail everything. TableDescriptor tableDescriptor = util.getAdmin().getDescriptor(tableName); TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableDescriptor); // Make the max file size huge so that splits don't happen during the test. builder.setMaxFileSize(Long.MAX_VALUE); ColumnFamilyDescriptorBuilder colDescriptorBldr = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY); colDescriptorBldr.setMaxVersions(1); builder.setColumnFamily(colDescriptorBldr.build()); util.getAdmin().createTable(builder.build()); // Setup the table for LoadTestTool int ret = loadTool.run(new String[]{"-tn", loadTableName.getNameAsString(), "-init_only"}); assertEquals("Failed to initialize LoadTestTool", 0, ret); }
@Test public void testCacheBlocks() throws IOException { // Set index block size to be the same as normal block size. TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, BLOCK_SIZE); BlockCache blockCache = BlockCacheFactory.createBlockCache(TEST_UTIL.getConfiguration()); ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CF)).setMaxVersions(MAX_VERSIONS) .setCompressionType(COMPRESSION_ALGORITHM).setBloomFilterType(BLOOM_TYPE) .setBlocksize(BLOCK_SIZE).setBlockCacheEnabled(cfCacheEnabled).build(); HRegion region = TEST_UTIL.createTestRegion(TABLE, cfd, blockCache); CacheStats stats = blockCache.getStats(); writeTestData(region); assertEquals(0, stats.getHitCount()); assertEquals(0, HFile.DATABLOCK_READ_COUNT.sum()); // Do a single get, take count of caches. If we are NOT caching DATA blocks, the miss // count should go up. Otherwise, all should be cached and the miss count should not rise. region.get(new Get(Bytes.toBytes("row" + 0))); assertTrue(stats.getHitCount() > 0); assertTrue(HFile.DATABLOCK_READ_COUNT.sum() > 0); long missCount = stats.getMissCount(); region.get(new Get(Bytes.toBytes("row" + 0))); if (this.cfCacheEnabled) assertEquals(missCount, stats.getMissCount()); else assertTrue(stats.getMissCount() > missCount); }
private static HRegion initHRegion(TableName tableName, byte[] startKey, byte[] stopKey, String callingMethod, Configuration conf, HBaseTestingUtility testUtil, boolean isReadOnly, byte[]... families) throws IOException { RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).setStartKey(startKey).setEndKey(stopKey).build(); TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName); builder.setReadOnly(isReadOnly).setDurability(Durability.SYNC_WAL); for (byte[] family : families) { builder.setColumnFamily( ColumnFamilyDescriptorBuilder.newBuilder(family).setMaxVersions(Integer.MAX_VALUE) .build()); } return HBaseTestingUtility .createRegionAndWAL(regionInfo, testUtil.getDataTestDir(callingMethod), conf, builder.build(), BlockCacheFactory.createBlockCache(conf)); } }