protected HTableDescriptor getTableDescriptor() { if (TABLE_DESCRIPTOR == null) { TABLE_DESCRIPTOR = new HTableDescriptor(tableName); HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME); family.setDataBlockEncoding(blockEncoding); family.setCompressionType(compression); if (inMemoryCF) { family.setInMemory(true); } TABLE_DESCRIPTOR.addFamily(family); } return TABLE_DESCRIPTOR; }
private void setupMockColumnFamiliesForDataBlockEncoding(Table table, Map<String, DataBlockEncoding> familyToDataBlockEncoding) throws IOException { HTableDescriptor mockTableDescriptor = new HTableDescriptor(TABLE_NAMES[0]); for (Entry<String, DataBlockEncoding> entry : familyToDataBlockEncoding.entrySet()) { mockTableDescriptor.addFamily(new HColumnDescriptor(entry.getKey()) .setMaxVersions(1) .setDataBlockEncoding(entry.getValue()) .setBlockCacheEnabled(false) .setTimeToLive(0)); } Mockito.doReturn(mockTableDescriptor).when(table).getTableDescriptor(); }
@Before public void setUp() throws Exception { TableName tableName = TableName.valueOf(getClass().getSimpleName()); HTableDescriptor htd = new HTableDescriptor(tableName); htd.addFamily(new HColumnDescriptor(cfName).setDataBlockEncoding(DataBlockEncoding.FAST_DIFF)); HRegionInfo info = new HRegionInfo(tableName, null, null, false); Path path = testUtil.getDataTestDir(getClass().getSimpleName()); region = HBaseTestingUtility.createRegionAndWAL(info, path, testUtil.getConfiguration(), htd); }
@Test public void testBlocksScannedWithEncoding() throws Exception { byte [] tableName = Bytes.toBytes("TestBlocksScannedWithEncoding"); HTableDescriptor table = new HTableDescriptor(TableName.valueOf(tableName)); table.addFamily( new HColumnDescriptor(FAMILY) .setMaxVersions(10) .setBlockCacheEnabled(true) .setDataBlockEncoding(DataBlockEncoding.FAST_DIFF) .setBlocksize(BLOCK_SIZE) .setCompressionType(Compression.Algorithm.NONE) ); _testBlocksScanned(table); }
String encodingStr = kylinConfig.getHbaseDefaultEncoding(); DataBlockEncoding encoding = DataBlockEncoding.valueOf(encodingStr); cf.setDataBlockEncoding(encoding); } catch (Exception e) { logger.info("hbase will not use any encoding", e); cf.setDataBlockEncoding(DataBlockEncoding.NONE);
public void setUp(long threshold, String TN, DataBlockEncoding encoding) throws Exception { desc = new HTableDescriptor(TableName.valueOf(TN)); hcd = new HColumnDescriptor(family); hcd.setMobEnabled(true); hcd.setMobThreshold(threshold); hcd.setMaxVersions(4); hcd.setDataBlockEncoding(encoding); desc.addFamily(hcd); admin = TEST_UTIL.getAdmin(); admin.createTable(desc); table = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()) .getTable(TableName.valueOf(TN)); }
/** * Creates a pre-split table for load testing. If the table already exists, * logs a warning and continues. * @return the number of regions the table was split into */ public static int createPreSplitLoadTestTable(Configuration conf, TableName tableName, byte[] columnFamily, Algorithm compression, DataBlockEncoding dataBlockEncoding, int numRegionsPerServer, int regionReplication, Durability durability) throws IOException { HTableDescriptor desc = new HTableDescriptor(tableName); desc.setDurability(durability); desc.setRegionReplication(regionReplication); HColumnDescriptor hcd = new HColumnDescriptor(columnFamily); hcd.setDataBlockEncoding(dataBlockEncoding); hcd.setCompressionType(compression); return createPreSplitLoadTestTable(conf, desc, hcd, numRegionsPerServer); }
private void createTable() throws Exception { deleteTable(); LOG.info("Creating table"); Configuration conf = util.getConfiguration(); String encodingKey = String.format(ENCODING_KEY, this.getClass().getSimpleName()); DataBlockEncoding blockEncoding = DataBlockEncoding.valueOf(conf.get(encodingKey, "FAST_DIFF")); HTableDescriptor htd = new HTableDescriptor(TABLE_NAME); for (byte[] cf : dataGen.getColumnFamilies()) { HColumnDescriptor hcd = new HColumnDescriptor(cf); hcd.setDataBlockEncoding(blockEncoding); htd.addFamily(hcd); } int serverCount = util.getHBaseClusterInterface().getClusterMetrics() .getLiveServerMetrics().size(); byte[][] splits = new RegionSplitter.HexStringSplit().split(serverCount * REGIONS_PER_SERVER); util.getAdmin().createTable(htd, splits); LOG.info("Created table"); }
/** * Create a set of column descriptors with the combination of compression, * encoding, bloom codecs available. * @param prefix family names prefix * @return the list of column descriptors */ public static List<HColumnDescriptor> generateColumnDescriptors(final String prefix) { List<HColumnDescriptor> htds = new ArrayList<>(); long familyId = 0; for (Compression.Algorithm compressionType: getSupportedCompressionAlgorithms()) { for (DataBlockEncoding encodingType: DataBlockEncoding.values()) { for (BloomType bloomType: BloomType.values()) { String name = String.format("%s-cf-!@#&-%d!@#", prefix, familyId); HColumnDescriptor htd = new HColumnDescriptor(name); htd.setCompressionType(compressionType); htd.setDataBlockEncoding(encodingType); htd.setBloomFilterType(bloomType); htds.add(htd); familyId++; } } } return htds; }
/** * Creates a pre-split table for load testing. If the table already exists, * logs a warning and continues. * @return the number of regions the table was split into */ public static int createPreSplitLoadTestTable(Configuration conf, TableName tableName, byte[][] columnFamilies, Algorithm compression, DataBlockEncoding dataBlockEncoding, int numRegionsPerServer, int regionReplication, Durability durability) throws IOException { HTableDescriptor desc = new HTableDescriptor(tableName); desc.setDurability(durability); desc.setRegionReplication(regionReplication); HColumnDescriptor[] hcds = new HColumnDescriptor[columnFamilies.length]; for (int i = 0; i < columnFamilies.length; i++) { HColumnDescriptor hcd = new HColumnDescriptor(columnFamilies[i]); hcd.setDataBlockEncoding(dataBlockEncoding); hcd.setCompressionType(compression); hcds[i] = hcd; } return createPreSplitLoadTestTable(conf, desc, hcds, numRegionsPerServer); }
/** * Create an HTableDescriptor from provided TestOptions. */ protected static HTableDescriptor getTableDescriptor(TestOptions opts) { HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(opts.tableName)); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); HColumnDescriptor familyDesc = new HColumnDescriptor(familyName); familyDesc.setDataBlockEncoding(opts.blockEncoding); familyDesc.setCompressionType(opts.compression); familyDesc.setBloomFilterType(opts.bloomType); familyDesc.setBlocksize(opts.blockSize); if (opts.inMemoryCF) { familyDesc.setInMemory(true); } familyDesc.setInMemoryCompaction(opts.inMemoryCompaction); tableDesc.addFamily(familyDesc); } if (opts.replicas != DEFAULT_OPTS.replicas) { tableDesc.setRegionReplication(opts.replicas); } if (opts.splitPolicy != null && !opts.splitPolicy.equals(DEFAULT_OPTS.splitPolicy)) { tableDesc.setRegionSplitPolicyClassName(opts.splitPolicy); } return tableDesc; }
for (byte[] family : families) { HColumnDescriptor hcd = new HColumnDescriptor(family); hcd.setDataBlockEncoding(blockEncoding); desc.addFamily(hcd);
private void createTableWithNonDefaultProperties() throws Exception { final long startTime = System.currentTimeMillis(); final String sourceTableNameAsString = STRING_TABLE_NAME + startTime; originalTableName = TableName.valueOf(sourceTableNameAsString); // enable replication on a column family HColumnDescriptor maxVersionsColumn = new HColumnDescriptor(MAX_VERSIONS_FAM); HColumnDescriptor bloomFilterColumn = new HColumnDescriptor(BLOOMFILTER_FAM); HColumnDescriptor dataBlockColumn = new HColumnDescriptor(COMPRESSED_FAM); HColumnDescriptor blockSizeColumn = new HColumnDescriptor(BLOCKSIZE_FAM); maxVersionsColumn.setMaxVersions(MAX_VERSIONS); bloomFilterColumn.setBloomFilterType(BLOOM_TYPE); dataBlockColumn.setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE); blockSizeColumn.setBlocksize(BLOCK_SIZE); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(sourceTableNameAsString)); htd.addFamily(maxVersionsColumn); htd.addFamily(bloomFilterColumn); htd.addFamily(dataBlockColumn); htd.addFamily(blockSizeColumn); htd.setValue(TEST_CUSTOM_VALUE, TEST_CUSTOM_VALUE); htd.setConfiguration(TEST_CONF_CUSTOM_VALUE, TEST_CONF_CUSTOM_VALUE); assertTrue(htd.getConfiguration().size() > 0); admin.createTable(htd); Table original = UTIL.getConnection().getTable(originalTableName); originalTableName = TableName.valueOf(sourceTableNameAsString); originalTableDescriptor = admin.getTableDescriptor(originalTableName); originalTableDescription = originalTableDescriptor.toStringCustomizedValues(); original.close(); }
protected void prepareForLoadTest() throws IOException { LOG.info("Starting load test: dataBlockEncoding=" + dataBlockEncoding + ", isMultiPut=" + isMultiPut); numKeys = numKeys(); Admin admin = TEST_UTIL.getAdmin(); while (admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) .getLiveServerMetrics().size() < NUM_RS) { LOG.info("Sleeping until " + NUM_RS + " RSs are online"); Threads.sleepWithoutInterrupt(1000); } admin.close(); HTableDescriptor htd = new HTableDescriptor(TABLE); HColumnDescriptor hcd = new HColumnDescriptor(CF) .setCompressionType(compression) .setDataBlockEncoding(dataBlockEncoding); createPreSplitLoadTestTable(htd, hcd); LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator(CF); writerThreads = prepareWriterThreads(dataGen, conf, TABLE); readerThreads = prepareReaderThreads(dataGen, conf, TABLE, 100); }
HColumnDescriptor colDesc = new HColumnDescriptor(fam); colDesc.setBlockCacheEnabled(true); colDesc.setDataBlockEncoding(DataBlockEncoding.NONE); desc.addFamily(colDesc); Admin admin = TEST_UTIL.getAdmin();
private void setEncodingConf(DataBlockEncoding encoding, boolean onlineChange) throws Exception { LOG.debug("Setting CF encoding to " + encoding + " (ordinal=" + encoding.ordinal() + "), onlineChange=" + onlineChange); hcd.setDataBlockEncoding(encoding); try (Admin admin = TEST_UTIL.getConnection().getAdmin()) { if (!onlineChange) { admin.disableTable(tableName); } admin.modifyColumnFamily(tableName, hcd); if (!onlineChange) { admin.enableTable(tableName); } } // This is a unit test, not integration test. So let's // wait for regions out of transition. Otherwise, for online // encoding change, verification phase may be flaky because // regions could be still in transition. TEST_UTIL.waitUntilNoRegionsInTransition(TIMEOUT_MS); }
HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(FAMILY); hcd.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF); desc.addFamily(hcd); TEST_UTIL.getAdmin().createTable(desc);
HColumnDescriptor colDesc = new HColumnDescriptor(fam); colDesc.setBlockCacheEnabled(true); colDesc.setDataBlockEncoding(encoding); desc.addFamily(colDesc); Admin admin = TEST_UTIL.getAdmin();
private HColumnDescriptor generateColumnFamilyDescriptor(Pair<byte[],Map<String,Object>> family, PTableType tableType) throws SQLException { HColumnDescriptor columnDesc = new HColumnDescriptor(family.getFirst()); if (tableType != PTableType.VIEW) { columnDesc.setKeepDeletedCells(true); columnDesc.setDataBlockEncoding(SchemaUtil.DEFAULT_DATA_BLOCK_ENCODING); for (Entry<String,Object> entry : family.getSecond().entrySet()) { String key = entry.getKey(); Object value = entry.getValue(); columnDesc.setValue(key, value == null ? null : value.toString()); } } return columnDesc; }
boolean inmemory = hcd.isInMemory(); hcd.setScope(v); hcd.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF); hcd.setBloomFilterType(BloomType.ROW); hcd.setCompressionType(Algorithm.SNAPPY);