if (contains(getMethod2Value.values(), str)) { LOG.info("Set to lower case " + str.toLowerCase()); builder2.setValue(k, new Bytes(Bytes.toBytes(str.toLowerCase())));
/** * Converts a ColumnFamilySchema to ColumnFamilyDescriptor * @param cfs the ColumnFamilySchema * @return An {@link ColumnFamilyDescriptor} made from the passed in <code>cfs</code> */ public static ColumnFamilyDescriptor toColumnFamilyDescriptor(final ColumnFamilySchema cfs) { // Use the empty constructor so we preserve the initial values set on construction for things // like maxVersion. Otherwise, we pick up wrong values on deserialization which makes for // unrelated-looking test failures that are hard to trace back to here. ColumnFamilyDescriptorBuilder builder = ColumnFamilyDescriptorBuilder.newBuilder(cfs.getName().toByteArray()); cfs.getAttributesList().forEach(a -> builder.setValue(a.getFirst().toByteArray(), a.getSecond().toByteArray())); cfs.getConfigurationList().forEach(a -> builder.setConfiguration(a.getName(), a.getValue())); return builder.build(); }
@Test public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception { final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle"; long anyValue = 10; // We'll check that it uses correct config and propagates it appropriately by going thru // the simplest "real" path I can find - "throttleCompaction", which just checks whether // a number we pass in is higher than some config value, inside compactionPolicy. Configuration conf = HBaseConfiguration.create(); conf.setLong(CONFIG_KEY, anyValue); init(name.getMethodName() + "-xml", conf); assertTrue(store.throttleCompaction(anyValue + 1)); assertFalse(store.throttleCompaction(anyValue)); // HTD overrides XML. --anyValue; init(name.getMethodName() + "-htd", conf, TableDescriptorBuilder .newBuilder(TableName.valueOf(table)).setValue(CONFIG_KEY, Long.toString(anyValue)), ColumnFamilyDescriptorBuilder.of(family)); assertTrue(store.throttleCompaction(anyValue + 1)); assertFalse(store.throttleCompaction(anyValue)); // HCD overrides them both. --anyValue; init(name.getMethodName() + "-hcd", conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setValue(CONFIG_KEY, Long.toString(anyValue)), ColumnFamilyDescriptorBuilder.newBuilder(family).setValue(CONFIG_KEY, Long.toString(anyValue)) .build()); assertTrue(store.throttleCompaction(anyValue + 1)); assertFalse(store.throttleCompaction(anyValue)); }
builder.setValue(attribute.getKey().array(), attribute.getValue().array());
private static void setHColumnDescriptorValue(ColumnFamilyDescriptorBuilder columnDescBuilder, String key, Object value) { if (HConstants.VERSIONS.equals(key)) { columnDescBuilder.setMaxVersions(getMaxVersion(value)); } else { columnDescBuilder.setValue(key, value == null ? null : value.toString()); } }
/** * Synchronize column family properties using the default cf properties for a given table * @param tableDesc table descriptor of table to modify * @param defaultColFam default column family used as the baseline for property synchronization * @param syncedProps Map of properties to be kept in sync as read from the default column family descriptor * @return modified table descriptor builder */ private static TableDescriptorBuilder syncColFamProperties(TableDescriptor tableDesc, ColumnFamilyDescriptor defaultColFam, Map<String, Object> syncedProps) { TableDescriptorBuilder tableDescBuilder = TableDescriptorBuilder.newBuilder(tableDesc); // Ensure that all column families have necessary properties in sync (including local index cf if present) for (ColumnFamilyDescriptor currentColFam: tableDesc.getColumnFamilies()) { if (!currentColFam.equals(defaultColFam)) { ColumnFamilyDescriptorBuilder colFamDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(currentColFam); for (String prop: MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) { String existingPropVal = Bytes.toString(currentColFam.getValue(Bytes.toBytes(prop))); String expectedPropVal = syncedProps.get(prop).toString(); if (existingPropVal == null || !existingPropVal.toLowerCase().equals(expectedPropVal.toLowerCase())) { // Need to synchronize this property for the current column family descriptor colFamDescBuilder.setValue(prop, expectedPropVal); } } if (!colFamDescBuilder.equals(ColumnFamilyDescriptorBuilder.newBuilder(currentColFam))) { tableDescBuilder.modifyColumnFamily(colFamDescBuilder.build()); } } } return tableDescBuilder; }
builder.setTimeToLive(v); builder.setBlockCacheEnabled(!HColumnDescriptor.DEFAULT_BLOCKCACHE); builder.setValue(Bytes.toBytes("a"), Bytes.toBytes("b")); builder.setMaxVersions(v); assertEquals(v, builder.build().getMaxVersions());
ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(localIndexCf)); for(Entry<Bytes, Bytes> keyValue: cf.getValues().entrySet()){ colDefBuilder.setValue(keyValue.getKey().copyBytes(), keyValue.getValue().copyBytes());
/** * Converts a ColumnFamilySchema to ColumnFamilyDescriptor * @param cfs the ColumnFamilySchema * @return An {@link ColumnFamilyDescriptor} made from the passed in <code>cfs</code> */ public static ColumnFamilyDescriptor toColumnFamilyDescriptor(final ColumnFamilySchema cfs) { // Use the empty constructor so we preserve the initial values set on construction for things // like maxVersion. Otherwise, we pick up wrong values on deserialization which makes for // unrelated-looking test failures that are hard to trace back to here. ColumnFamilyDescriptorBuilder builder = ColumnFamilyDescriptorBuilder.newBuilder(cfs.getName().toByteArray()); cfs.getAttributesList().forEach(a -> builder.setValue(a.getFirst().toByteArray(), a.getSecond().toByteArray())); cfs.getConfigurationList().forEach(a -> builder.setConfiguration(a.getName(), a.getValue())); return builder.build(); }
private void setSharedIndexMaxVersion(PTable table, TableDescriptor tableDescriptor, TableDescriptorBuilder indexDescriptorBuilder) { if (table.getColumnFamilies().isEmpty()) { byte[] familyName = SchemaUtil.getEmptyColumnFamily(table); ColumnFamilyDescriptorBuilder indexColDescriptorBuilder = ColumnFamilyDescriptorBuilder.newBuilder(indexDescriptorBuilder.build().getColumnFamily(familyName)); ColumnFamilyDescriptor tableColDescriptor = tableDescriptor.getColumnFamily(familyName); indexColDescriptorBuilder.setMaxVersions(tableColDescriptor.getMaxVersions()); indexColDescriptorBuilder.setValue( Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL),tableColDescriptor.getValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL))); indexDescriptorBuilder.removeColumnFamily(familyName); indexDescriptorBuilder.addColumnFamily(indexColDescriptorBuilder.build()); } else { for (PColumnFamily family : table.getColumnFamilies()) { byte[] familyName = family.getName().getBytes(); ColumnFamilyDescriptor indexColDescriptor = indexDescriptorBuilder.build().getColumnFamily(familyName); if (indexColDescriptor != null) { ColumnFamilyDescriptor tableColDescriptor = tableDescriptor.getColumnFamily(familyName); ColumnFamilyDescriptorBuilder indexColDescriptorBuilder = ColumnFamilyDescriptorBuilder.newBuilder(indexColDescriptor); indexColDescriptorBuilder.setMaxVersions(tableColDescriptor.getMaxVersions()); indexColDescriptorBuilder.setValue( Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL),tableColDescriptor.getValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL))); indexDescriptorBuilder.removeColumnFamily(familyName); indexDescriptorBuilder.addColumnFamily(indexColDescriptorBuilder.build()); } } } }
ColumnFamilyDescriptor tableColDescriptor = tableDescriptorBuilder.build().getColumnFamily(dataFamilyName); indexColDescriptor.setMaxVersions(tableColDescriptor.getMaxVersions()); indexColDescriptor.setValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL), tableColDescriptor.getValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL))); indexDescriptorBuilder.removeColumnFamily(indexFamilyName); ColumnFamilyDescriptor tableColDescriptor = tableDescriptorBuilder.build().getColumnFamily(familyName); indexColDescriptor.setMaxVersions(tableColDescriptor.getMaxVersions()); indexColDescriptor.setValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL), tableColDescriptor.getValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL))); indexDescriptorBuilder.removeColumnFamily(familyName);
private static void setHColumnDescriptorValue(ColumnFamilyDescriptorBuilder columnDescBuilder, String key, Object value) { if (HConstants.VERSIONS.equals(key)) { columnDescBuilder.setMaxVersions(getMaxVersion(value)); } else { columnDescBuilder.setValue(key, value == null ? null : value.toString()); } }
private static void setHColumnDescriptorValue(ColumnFamilyDescriptorBuilder columnDescBuilder, String key, Object value) { if (HConstants.VERSIONS.equals(key)) { columnDescBuilder.setMaxVersions(getMaxVersion(value)); } else { columnDescBuilder.setValue(key, value == null ? null : value.toString()); } }
/** * Synchronize column family properties using the default cf properties for a given table * @param tableDesc table descriptor of table to modify * @param defaultColFam default column family used as the baseline for property synchronization * @param syncedProps Map of properties to be kept in sync as read from the default column family descriptor * @return modified table descriptor builder */ private static TableDescriptorBuilder syncColFamProperties(TableDescriptor tableDesc, ColumnFamilyDescriptor defaultColFam, Map<String, Object> syncedProps) { TableDescriptorBuilder tableDescBuilder = TableDescriptorBuilder.newBuilder(tableDesc); // Ensure that all column families have necessary properties in sync (including local index cf if present) for (ColumnFamilyDescriptor currentColFam: tableDesc.getColumnFamilies()) { if (!currentColFam.equals(defaultColFam)) { ColumnFamilyDescriptorBuilder colFamDescBuilder = ColumnFamilyDescriptorBuilder.newBuilder(currentColFam); for (String prop: MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) { String existingPropVal = Bytes.toString(currentColFam.getValue(Bytes.toBytes(prop))); String expectedPropVal = syncedProps.get(prop).toString(); if (existingPropVal == null || !existingPropVal.toLowerCase().equals(expectedPropVal.toLowerCase())) { // Need to synchronize this property for the current column family descriptor colFamDescBuilder.setValue(prop, expectedPropVal); } } if (!colFamDescBuilder.equals(ColumnFamilyDescriptorBuilder.newBuilder(currentColFam))) { tableDescBuilder.modifyColumnFamily(colFamDescBuilder.build()); } } } return tableDescBuilder; }
builder.setTimeToLive(v); builder.setBlockCacheEnabled(!HColumnDescriptor.DEFAULT_BLOCKCACHE); builder.setValue(Bytes.toBytes("a"), Bytes.toBytes("b")); builder.setMaxVersions(v); assertEquals(v, builder.build().getMaxVersions());
/** * Converts a ColumnFamilySchema to ColumnFamilyDescriptor * @param cfs the ColumnFamilySchema * @return An {@link ColumnFamilyDescriptor} made from the passed in <code>cfs</code> */ public static ColumnFamilyDescriptor toColumnFamilyDescriptor(final ColumnFamilySchema cfs) { // Use the empty constructor so we preserve the initial values set on construction for things // like maxVersion. Otherwise, we pick up wrong values on deserialization which makes for // unrelated-looking test failures that are hard to trace back to here. ColumnFamilyDescriptorBuilder builder = ColumnFamilyDescriptorBuilder.newBuilder(cfs.getName().toByteArray()); cfs.getAttributesList().forEach(a -> builder.setValue(a.getFirst().toByteArray(), a.getSecond().toByteArray())); cfs.getConfigurationList().forEach(a -> builder.setConfiguration(a.getName(), a.getValue())); return builder.build(); }
private void setSharedIndexMaxVersion(PTable table, TableDescriptor tableDescriptor, TableDescriptorBuilder indexDescriptorBuilder) { if (table.getColumnFamilies().isEmpty()) { byte[] familyName = SchemaUtil.getEmptyColumnFamily(table); ColumnFamilyDescriptorBuilder indexColDescriptorBuilder = ColumnFamilyDescriptorBuilder.newBuilder(indexDescriptorBuilder.build().getColumnFamily(familyName)); ColumnFamilyDescriptor tableColDescriptor = tableDescriptor.getColumnFamily(familyName); indexColDescriptorBuilder.setMaxVersions(tableColDescriptor.getMaxVersions()); indexColDescriptorBuilder.setValue( Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL),tableColDescriptor.getValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL))); indexDescriptorBuilder.removeColumnFamily(familyName); indexDescriptorBuilder.addColumnFamily(indexColDescriptorBuilder.build()); } else { for (PColumnFamily family : table.getColumnFamilies()) { byte[] familyName = family.getName().getBytes(); ColumnFamilyDescriptor indexColDescriptor = indexDescriptorBuilder.build().getColumnFamily(familyName); if (indexColDescriptor != null) { ColumnFamilyDescriptor tableColDescriptor = tableDescriptor.getColumnFamily(familyName); ColumnFamilyDescriptorBuilder indexColDescriptorBuilder = ColumnFamilyDescriptorBuilder.newBuilder(indexColDescriptor); indexColDescriptorBuilder.setMaxVersions(tableColDescriptor.getMaxVersions()); indexColDescriptorBuilder.setValue( Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL),tableColDescriptor.getValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL))); indexDescriptorBuilder.removeColumnFamily(familyName); indexDescriptorBuilder.addColumnFamily(indexColDescriptorBuilder.build()); } } } }
private void setSharedIndexMaxVersion(PTable table, TableDescriptor tableDescriptor, TableDescriptorBuilder indexDescriptorBuilder) { if (table.getColumnFamilies().isEmpty()) { byte[] familyName = SchemaUtil.getEmptyColumnFamily(table); ColumnFamilyDescriptorBuilder indexColDescriptorBuilder = ColumnFamilyDescriptorBuilder.newBuilder(indexDescriptorBuilder.build().getColumnFamily(familyName)); ColumnFamilyDescriptor tableColDescriptor = tableDescriptor.getColumnFamily(familyName); indexColDescriptorBuilder.setMaxVersions(tableColDescriptor.getMaxVersions()); indexColDescriptorBuilder.setValue( Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL),tableColDescriptor.getValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL))); indexDescriptorBuilder.removeColumnFamily(familyName); indexDescriptorBuilder.addColumnFamily(indexColDescriptorBuilder.build()); } else { for (PColumnFamily family : table.getColumnFamilies()) { byte[] familyName = family.getName().getBytes(); ColumnFamilyDescriptor indexColDescriptor = indexDescriptorBuilder.build().getColumnFamily(familyName); if (indexColDescriptor != null) { ColumnFamilyDescriptor tableColDescriptor = tableDescriptor.getColumnFamily(familyName); ColumnFamilyDescriptorBuilder indexColDescriptorBuilder = ColumnFamilyDescriptorBuilder.newBuilder(indexColDescriptor); indexColDescriptorBuilder.setMaxVersions(tableColDescriptor.getMaxVersions()); indexColDescriptorBuilder.setValue( Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL),tableColDescriptor.getValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL))); indexDescriptorBuilder.removeColumnFamily(familyName); indexDescriptorBuilder.addColumnFamily(indexColDescriptorBuilder.build()); } } } }
ColumnFamilyDescriptor tableColDescriptor = tableDescriptorBuilder.build().getColumnFamily(dataFamilyName); indexColDescriptor.setMaxVersions(tableColDescriptor.getMaxVersions()); indexColDescriptor.setValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL), tableColDescriptor.getValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL))); indexDescriptorBuilder.removeColumnFamily(indexFamilyName); ColumnFamilyDescriptor tableColDescriptor = tableDescriptorBuilder.build().getColumnFamily(familyName); indexColDescriptor.setMaxVersions(tableColDescriptor.getMaxVersions()); indexColDescriptor.setValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL), tableColDescriptor.getValue(Bytes.toBytes(PhoenixTransactionContext.PROPERTY_TTL))); indexDescriptorBuilder.removeColumnFamily(familyName);
@Test public void testStoreUsesConfigurationFromHcdAndHtd() throws Exception { final String CONFIG_KEY = "hbase.regionserver.thread.compaction.throttle"; long anyValue = 10; // We'll check that it uses correct config and propagates it appropriately by going thru // the simplest "real" path I can find - "throttleCompaction", which just checks whether // a number we pass in is higher than some config value, inside compactionPolicy. Configuration conf = HBaseConfiguration.create(); conf.setLong(CONFIG_KEY, anyValue); init(name.getMethodName() + "-xml", conf); assertTrue(store.throttleCompaction(anyValue + 1)); assertFalse(store.throttleCompaction(anyValue)); // HTD overrides XML. --anyValue; init(name.getMethodName() + "-htd", conf, TableDescriptorBuilder .newBuilder(TableName.valueOf(table)).setValue(CONFIG_KEY, Long.toString(anyValue)), ColumnFamilyDescriptorBuilder.of(family)); assertTrue(store.throttleCompaction(anyValue + 1)); assertFalse(store.throttleCompaction(anyValue)); // HCD overrides them both. --anyValue; init(name.getMethodName() + "-hcd", conf, TableDescriptorBuilder.newBuilder(TableName.valueOf(table)).setValue(CONFIG_KEY, Long.toString(anyValue)), ColumnFamilyDescriptorBuilder.newBuilder(family).setValue(CONFIG_KEY, Long.toString(anyValue)) .build()); assertTrue(store.throttleCompaction(anyValue + 1)); assertFalse(store.throttleCompaction(anyValue)); }