/** * Serialize column family to block size map to configuration. * Invoked while configuring the MR job for incremental load. * @param tableDescriptor to read the properties from * @param conf to persist serialized values into * * @throws IOException * on failure to read column family descriptors */ @VisibleForTesting static void configureBlockSize(HTableDescriptor tableDescriptor, Configuration conf) throws UnsupportedEncodingException { StringBuilder blockSizeConfigValue = new StringBuilder(); if (tableDescriptor == null) { // could happen with mock table instance return; } Collection<HColumnDescriptor> families = tableDescriptor.getFamilies(); int i = 0; for (HColumnDescriptor familyDescriptor : families) { if (i++ > 0) { blockSizeConfigValue.append('&'); } blockSizeConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8")); blockSizeConfigValue.append('='); blockSizeConfigValue.append(URLEncoder.encode(String.valueOf(familyDescriptor.getBlocksize()), "UTF-8")); } // Get rid of the last ampersand conf.set(BLOCK_SIZE_FAMILIES_CONF_KEY, blockSizeConfigValue.toString()); }
hof.getOutputCommitter(context).commitJob(context); FileStatus[] families = FSUtils.listStatus(fs, dir, new FSUtils.FamilyDirFilter(fs)); assertEquals(htd.getFamilies().size(), families.length); for (FileStatus f : families) { String familyStr = f.getPath().getName();
Collection<HColumnDescriptor> families = tableDescriptor.getFamilies(); int i = 0; for (HColumnDescriptor familyDescriptor : families) {
scopes.put(fam, 0); for (HColumnDescriptor hcd: htd.getFamilies()) { addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee, wal1, htd, mvcc, scopes); for (HColumnDescriptor hcd: htd.getFamilies()) { addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee, wal2, htd, mvcc, scopes);
Collection<HColumnDescriptor> families = tableDescriptor.getFamilies(); int i = 0; for (HColumnDescriptor familyDescriptor : families) {
Collection<HColumnDescriptor> families = tableDescriptor.getFamilies(); int i = 0; for (HColumnDescriptor familyDescriptor : families) {
@Override public Map<ByteBuffer, ColumnDescriptor> getColumnDescriptors( ByteBuffer tableName) throws IOError, TException { Table table = null; try { TreeMap<ByteBuffer, ColumnDescriptor> columns = new TreeMap<>(); table = getTable(tableName); HTableDescriptor desc = table.getTableDescriptor(); for (HColumnDescriptor e : desc.getFamilies()) { ColumnDescriptor col = ThriftUtilities.colDescFromHbase(e); columns.put(col.name, col); } return columns; } catch (IOException e) { LOG.warn(e.getMessage(), e); throw getIOError(e); } finally { closeTable(table); } }
/** * Constructor * @param htd the table descriptor */ public TableSchemaModel(HTableDescriptor htd) { setName(htd.getTableName().getNameAsString()); for (Map.Entry<Bytes, Bytes> e: htd.getValues().entrySet()) { addAttribute(Bytes.toString(e.getKey().get()), Bytes.toString(e.getValue().get())); } for (HColumnDescriptor hcd: htd.getFamilies()) { ColumnSchemaModel columnModel = new ColumnSchemaModel(); columnModel.setName(hcd.getNameAsString()); for (Map.Entry<Bytes, Bytes> e: hcd.getValues().entrySet()) { columnModel.addAttribute(Bytes.toString(e.getKey().get()), Bytes.toString(e.getValue().get())); } addColumnFamily(columnModel); } }
private void createTableAndFlush(HTableDescriptor htd) throws Exception { HColumnDescriptor hcd = htd.getFamilies().iterator().next(); // Create the test table TEST_UTIL.getAdmin().createTable(htd); TEST_UTIL.waitTableAvailable(htd.getTableName(), 5000); // Create a store file Table table = TEST_UTIL.getConnection().getTable(htd.getTableName()); try { table.put(new Put(Bytes.toBytes("testrow")) .addColumn(hcd.getName(), Bytes.toBytes("q"), Bytes.toBytes("value"))); } finally { table.close(); } TEST_UTIL.getAdmin().flush(htd.getTableName()); }
@Test public void testDefaultInMemoryCompaction() { PerformanceEvaluation.TestOptions defaultOpts = new PerformanceEvaluation.TestOptions(); assertEquals(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_DEFAULT.toString(), defaultOpts.getInMemoryCompaction().toString()); HTableDescriptor htd = PerformanceEvaluation.getTableDescriptor(defaultOpts); for (HColumnDescriptor hcd: htd.getFamilies()) { assertEquals(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_DEFAULT.toString(), hcd.getInMemoryCompaction().toString()); } }
Result result = region.get(get); assertEquals(countPerFamily * (htd.getFamilies().size() - 1), result.size()); region.close();
@Override public Object run() throws Exception { runWALSplit(newConf); FileSystem newFS = FileSystem.get(newConf); // Make a new wal for new region open. WAL wal3 = createWAL(newConf, hbaseRootDir, logName); final AtomicInteger countOfRestoredEdits = new AtomicInteger(0); HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) { @Override protected void restoreEdit(HStore s, Cell cell, MemStoreSizing memstoreSizing) { super.restoreEdit(s, cell, memstoreSizing); countOfRestoredEdits.incrementAndGet(); } }; long seqid3 = region3.initialize(); Result result3 = region3.get(g); // Assert that count of cells is same as before crash. assertEquals(result2.size(), result3.size()); assertEquals(htd.getFamilies().size() * countPerFamily, countOfRestoredEdits.get()); // I can't close wal1. Its been appropriated when we split. region3.close(); wal3.close(); return null; } });
HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null); int writtenRowCount = 10; List<HColumnDescriptor> families = new ArrayList<>(htd.getFamilies()); for (int i = 0; i < writtenRowCount; i++) { Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf); byte [] family = htd.getFamilies().iterator().next().getName(); Path f = new Path(basedir, "hfile"); HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(""),
byte [] family = htd.getFamilies().iterator().next().getName(); region.put((new Put(row)).addColumn(family, family, family)); wal.sync();
for (HColumnDescriptor hcd : htd.getFamilies()) { addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x"); for (HColumnDescriptor hcd : htd.getFamilies()) { addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
for (HColumnDescriptor hcd : htd.getFamilies()) { addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x"); assertEquals(countPerFamily * htd.getFamilies().size(), result.size());
for (HColumnDescriptor c : desc.getFamilies()) c.setValue(attrName, attrValue); Bytes.compareTo(value, attrValue) != 0); for (HColumnDescriptor c : desc.getFamilies()) { value = c.getValue(attrName); assertFalse("missing HCD attribute value", value == null);
HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal); long seqid = region.getOpenSeqNum(); for (HColumnDescriptor hcd: htd.getFamilies()) { addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x"); assertEquals(countPerFamily * htd.getFamilies().size(), result.size()); for (HColumnDescriptor hcd: htd.getFamilies()) { cf_count++; if (cf_count == 2) {
int countOfFamilies = modifiedHtd.getFamilies().size(); assertTrue(countOfFamilies > 0); HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next(); int maxversions = hcd.getMaxVersions(); final int newMaxVersions = maxversions + 1;