/** * This method recreates and registers all {@link ColumnFamilyDescriptor} from Flink's state meta data snapshot. */ private List<ColumnFamilyDescriptor> createAndRegisterColumnFamilyDescriptors( List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) { List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(stateMetaInfoSnapshots.size()); for (StateMetaInfoSnapshot stateMetaInfoSnapshot : stateMetaInfoSnapshots) { ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor( stateMetaInfoSnapshot.getName().getBytes(ConfigConstants.DEFAULT_CHARSET), stateBackend.columnOptions); columnFamilyDescriptors.add(columnFamilyDescriptor); } return columnFamilyDescriptors; }
private synchronized ColumnFamilyHandle getOrCreateColumnFamily(String kvStoreId) throws IOException { ColumnFamilyHandle columnFamily = columnFamilies.get(kvStoreId); if (columnFamily == null) { ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(kvStoreId.getBytes(), columnOptions); try { columnFamily = rocksDB.createColumnFamily(columnDescriptor); columnFamilies.put(kvStoreId, columnFamily); } catch (RocksDBException e) { throw new IOException("Error creating ColumnFamilyHandle.", e); } } return columnFamily; }
private ColumnFamilyHandle getOrCreateColumnFamily(String kvStoreId) throws IOException { ColumnFamilyHandle columnFamily = columnFamilies.get(kvStoreId); // TODO: verify existing column family is compatible with the request. if (columnFamily == null) { ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(kvStoreId.getBytes(), columnOptions); try { columnFamily = rocksDB.createColumnFamily(columnDescriptor); columnFamilies.put(kvStoreId, columnFamily); } catch (RocksDBException e) { throw new IOException("Error creating ColumnFamilyHandle.", e); } } return columnFamily; }
/** * Creates a column family handle for use with a k/v state. */ private ColumnFamilyHandle createColumnFamily(String stateName) { byte[] nameBytes = stateName.getBytes(ConfigConstants.DEFAULT_CHARSET); Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes), "The chosen state name 'default' collides with the name of the default column family!"); ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(nameBytes, columnOptions); try { return db.createColumnFamily(columnDescriptor); } catch (RocksDBException e) { throw new FlinkRuntimeException("Error creating ColumnFamilyHandle.", e); } }
private static List<ColumnFamilyDescriptor> getExistingColumnFamilyDesc(Map conf, String dbPath) throws IOException { try { List<byte[]> families = Lists.newArrayList(); List<byte[]> existingFamilies = RocksDB.listColumnFamilies(getOptions(conf), dbPath); if (existingFamilies != null) { families.addAll(existingFamilies); } else { families.add(RocksDB.DEFAULT_COLUMN_FAMILY); } List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(); for (byte[] bytes : families) { columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, getColumnFamilyOptions(conf))); LOG.info("Load column family of {}", new String(bytes)); } return columnFamilyDescriptors; } catch (RocksDBException e) { throw new IOException("Failed to retrieve existing column families.", e); } }
private void createColumnFamily(final String name) throws RocksDBException { COLUMN_FAMILY_LOCKS.putIfAbsent(name, new ReentrantLock()); final Lock l = COLUMN_FAMILY_LOCKS.get(name); l.lock(); try { if(!COLUMN_FAMILIES.containsKey(name)) { final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions().optimizeLevelStyleCompaction(); final ColumnFamilyHandle cfHandle = rocksDb.createColumnFamily( new ColumnFamilyDescriptor(name.getBytes(UTF_8), cfOptions) ); COLUMN_FAMILIES.put(name, new ColumnFamily(cfHandle, cfOptions)); } } finally { l.unlock(); } }
public void initDb(List<Integer> list) throws Exception { LOG.info("Begin to init rocksDB of {}", rootDir); DBOptions dbOptions = null; List<ColumnFamilyDescriptor> columnFamilyNames = new ArrayList<>(); columnFamilyNames.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); for (Integer timeout : list) { columnFamilyNames.add(new ColumnFamilyDescriptor(String.valueOf(timeout).getBytes())); } List<Integer> ttlValues = new ArrayList<>(); // Default column family with infinite TTL // NOTE that the first must be 0, RocksDB.java API has this limitation ttlValues.add(0); // new column family with list second ttl ttlValues.addAll(list); try { dbOptions = new DBOptions().setCreateMissingColumnFamilies(true).setCreateIfMissing(true); List<ColumnFamilyHandle> columnFamilyHandleList = new ArrayList<>(); ttlDB = TtlDB.open(dbOptions, rootDir, columnFamilyNames, columnFamilyHandleList, ttlValues, false); for (int i = 0; i < ttlValues.size(); i++) { windowHandlers.put(ttlValues.get(i), columnFamilyHandleList.get(i)); } LOG.info("Successfully init rocksDB of {}", rootDir); } finally { if (dbOptions != null) { dbOptions.dispose(); } } }
columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, columnOptions)); columnFamilyDescriptors.addAll(stateColumnFamilyDescriptors);
handler2 = handlers.get(2); } else { handler1 = db.createColumnFamily(new ColumnFamilyDescriptor("test1".getBytes())); handler2 = db.createColumnFamily(new ColumnFamilyDescriptor("test2".getBytes()));
new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts), new ColumnFamilyDescriptor(BLOCK_META_COLUMN.getBytes(), cfOpts), new ColumnFamilyDescriptor(BLOCK_LOCATIONS_COLUMN.getBytes(), cfOpts) );
@Override public void truncate() { try { rocksDB.dropColumnFamily(storeHandle); storeHandle.dispose(); storeHandle = rocksDB.createColumnFamily(new ColumnFamilyDescriptor(getName().getBytes(), storeOptions)); } catch (RocksDBException e) { throw new VoldemortException("Failed to truncate DB", e); } }
new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOpts), new ColumnFamilyDescriptor(INODES_COLUMN.getBytes(), cfOpts), new ColumnFamilyDescriptor(EDGES_COLUMN.getBytes(), cfOpts) );
final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions() .optimizeLevelStyleCompaction(); final ColumnFamilyDescriptor cfDescriptor = new ColumnFamilyDescriptor( cfName.getBytes(UTF_8), cfOptions
descriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions)); descriptors.add(new ColumnFamilyDescriptor(storeName.getBytes(), cfOptions)); List<ColumnFamilyHandle> handles = new ArrayList<ColumnFamilyHandle>();
byte[] nameBytes = restoredMetaInfo.getName().getBytes(ConfigConstants.DEFAULT_CHARSET); ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor( nameBytes, rocksDBKeyedStateBackend.columnOptions);
if (families != null) { for (byte[] bytes : families) { columnFamilyDescriptors.add(new ColumnFamilyDescriptor(bytes, familyOptions)); LOG.info("Load colum family of {}", new String(bytes));
@Override public void createTable(String table) throws RocksDBException { if (this.cfs.containsKey(table)) { return; } this.checkValid(); // Should we use options.setCreateMissingColumnFamilies() to create CF ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(table)); ColumnFamilyOptions options = cfd.columnFamilyOptions(); initOptions(this.conf, null, options, options); this.cfs.put(table, this.rocksdb.createColumnFamily(cfd)); ingestExternalFile(); }
ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(cf)); ColumnFamilyOptions options = cfd.columnFamilyOptions(); RocksDBStdSessions.initOptions(this.conf, null, options, options);
private List<ColumnFamilyDescriptor> createFamilyDescriptors( List<byte[]> columnFamilyNames, ColumnFamilyOptions columnFamilyOptions) { final List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(); if (columnFamilyNames != null && columnFamilyNames.size() > 0) { for (byte[] name : columnFamilyNames) { final ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(name, columnFamilyOptions); columnFamilyDescriptors.add(columnFamilyDescriptor); } } return columnFamilyDescriptors; }
@Before public void setUpStore() { ColumnFamilyHandle handle = rocksDBResource.get().getDefaultColumnFamily(); store = new RocksDBStore("test", new ColumnFamilyDescriptor("test".getBytes(UTF_8)), handle, rocksDBResource.get(), 4); // Making sure test is repeatable Random random = new Random(42); for(int i = 0; i < 1 << 16; i++ ) { store.put(newRandomValue(random), newRandomValue(random)); } store.put(specialKey, newRandomValue(random)); }