private synchronized ColumnFamilyHandle getOrCreateColumnFamily(String kvStoreId) throws IOException { ColumnFamilyHandle columnFamily = columnFamilies.get(kvStoreId); if (columnFamily == null) { ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(kvStoreId.getBytes(), columnOptions); try { columnFamily = rocksDB.createColumnFamily(columnDescriptor); columnFamilies.put(kvStoreId, columnFamily); } catch (RocksDBException e) { throw new IOException("Error creating ColumnFamilyHandle.", e); } } return columnFamily; }
private ColumnFamilyHandle getOrCreateColumnFamily(String kvStoreId) throws IOException { ColumnFamilyHandle columnFamily = columnFamilies.get(kvStoreId); // TODO: verify existing column family is compatible with the request. if (columnFamily == null) { ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(kvStoreId.getBytes(), columnOptions); try { columnFamily = rocksDB.createColumnFamily(columnDescriptor); columnFamilies.put(kvStoreId, columnFamily); } catch (RocksDBException e) { throw new IOException("Error creating ColumnFamilyHandle.", e); } } return columnFamily; }
/** * Creates a column family handle for use with a k/v state. */ private ColumnFamilyHandle createColumnFamily(String stateName) { byte[] nameBytes = stateName.getBytes(ConfigConstants.DEFAULT_CHARSET); Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes), "The chosen state name 'default' collides with the name of the default column family!"); ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(nameBytes, columnOptions); try { return db.createColumnFamily(columnDescriptor); } catch (RocksDBException e) { throw new FlinkRuntimeException("Error creating ColumnFamilyHandle.", e); } }
private void createColumnFamily(final String name) throws RocksDBException { COLUMN_FAMILY_LOCKS.putIfAbsent(name, new ReentrantLock()); final Lock l = COLUMN_FAMILY_LOCKS.get(name); l.lock(); try { if(!COLUMN_FAMILIES.containsKey(name)) { final ColumnFamilyOptions cfOptions = new ColumnFamilyOptions().optimizeLevelStyleCompaction(); final ColumnFamilyHandle cfHandle = rocksDb.createColumnFamily( new ColumnFamilyDescriptor(name.getBytes(UTF_8), cfOptions) ); COLUMN_FAMILIES.put(name, new ColumnFamily(cfHandle, cfOptions)); } } finally { l.unlock(); } }
handler2 = handlers.get(2); } else { handler1 = db.createColumnFamily(new ColumnFamilyDescriptor("test1".getBytes())); handler2 = db.createColumnFamily(new ColumnFamilyDescriptor("test2".getBytes()));
private ColumnFamilyHandle getOrRegisterColumnFamilyHandle( ColumnFamilyDescriptor columnFamilyDescriptor, ColumnFamilyHandle columnFamilyHandle, StateMetaInfoSnapshot stateMetaInfoSnapshot) throws RocksDBException { Tuple2<ColumnFamilyHandle, RegisteredStateMetaInfoBase> registeredStateMetaInfoEntry = stateBackend.kvStateInformation.get(stateMetaInfoSnapshot.getName()); if (null == registeredStateMetaInfoEntry) { // create a meta info for the state on restore; // this allows us to retain the state in future snapshots even if it wasn't accessed RegisteredStateMetaInfoBase stateMetaInfo = RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot); registeredStateMetaInfoEntry = new Tuple2<>( columnFamilyHandle != null ? columnFamilyHandle : stateBackend.db.createColumnFamily(columnFamilyDescriptor), stateMetaInfo); stateBackend.registerKvStateInformation( stateMetaInfoSnapshot.getName(), registeredStateMetaInfoEntry); } return registeredStateMetaInfoEntry.f0; }
@Override public void truncate() { try { rocksDB.dropColumnFamily(storeHandle); storeHandle.dispose(); storeHandle = rocksDB.createColumnFamily(new ColumnFamilyDescriptor(getName().getBytes(), storeOptions)); } catch (RocksDBException e) { throw new VoldemortException("Failed to truncate DB", e); } }
rocksDBKeyedStateBackend.columnOptions); ColumnFamilyHandle columnFamily = rocksDBKeyedStateBackend.db.createColumnFamily(columnFamilyDescriptor);
@Override public void createTable(String table) throws RocksDBException { if (this.cfs.containsKey(table)) { return; } this.checkValid(); // Should we use options.setCreateMissingColumnFamilies() to create CF ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(table)); ColumnFamilyOptions options = cfd.columnFamilyOptions(); initOptions(this.conf, null, options, options); this.cfs.put(table, this.rocksdb.createColumnFamily(cfd)); ingestExternalFile(); }
public synchronized void create(String tableName) throws RocksDBException { ColumnFamily cf = cfs.get(tableName); if (cf == null) { ColumnFamilyDescriptor descriptor = cfFactory.apply(tableName); ColumnFamilyHandle handle = db.createColumnFamily(descriptor); cfs.put(tableName, new ColumnFamily(0, handle)); } }
protected void createKeySpace(ByteArray handleName) { ColumnFamilyDescriptor cfDescriptor = new ColumnFamilyDescriptor(handleName.getBytes()); ColumnFamilyHandle cfHandle; try { cfHandle = this.rocksDB.createColumnFamily(cfDescriptor); } catch (RocksDBException ex) { throw new RuntimeException(ex); } cfHandles.put(handleName, cfHandle); dataBatches.put(handleName, new HashMap<>()); }
public synchronized void truncate(String tableName) throws InterruptedException, RocksDBException { ColumnFamily oldCf = cfs.get(tableName); if (oldCf == null) { throw new IllegalArgumentException("Table " + tableName + " does not exist."); } long newIndex = (oldCf.index + 1) % 2; String realTableName = String.format("%s__%d", tableName, newIndex); ColumnFamilyDescriptor descriptor = cfFactory.apply(realTableName); ColumnFamilyHandle handle = db.createColumnFamily(descriptor); cfs.put(tableName, new ColumnFamily(newIndex, handle)); while (oldCf.refCount.get() > 0) { Thread.sleep(10); } db.dropColumnFamily(oldCf.handle); oldCf.handle.dispose(); } }
/** * Creates a column family handle for use with a k/v state. */ private ColumnFamilyHandle createColumnFamily(String stateName) { byte[] nameBytes = stateName.getBytes(ConfigConstants.DEFAULT_CHARSET); Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes), "The chosen state name 'default' collides with the name of the default column family!"); ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(nameBytes, columnOptions); try { return db.createColumnFamily(columnDescriptor); } catch (RocksDBException e) { throw new FlinkRuntimeException("Error creating ColumnFamilyHandle.", e); } }
/** * Creates a column family handle for use with a k/v state. */ private ColumnFamilyHandle createColumnFamily(String stateName) { byte[] nameBytes = stateName.getBytes(ConfigConstants.DEFAULT_CHARSET); Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes), "The chosen state name 'default' collides with the name of the default column family!"); ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(nameBytes, columnOptions); try { return db.createColumnFamily(columnDescriptor); } catch (RocksDBException e) { throw new FlinkRuntimeException("Error creating ColumnFamilyHandle.", e); } }
private ByteStore newStore(String name) throws RocksDBException { if (inMemory) { return new MapStore(name); } else { final ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor(name.getBytes(UTF_8)); ColumnFamilyHandle handle = db.createColumnFamily(columnFamilyDescriptor); handleIdToNameMap.put(handle.getID(), name); metadataManager.createEntry(name, false); return new RocksDBStore(name, columnFamilyDescriptor, handle, db, stripeCount); } }
@Override public void createTable(String table) throws RocksDBException { if (this.cfs.containsKey(table)) { return; } this.checkValid(); // Should we use options.setCreateMissingColumnFamilies() to create CF ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(table)); ColumnFamilyOptions options = cfd.columnFamilyOptions(); initOptions(this.conf, null, options, options); this.cfs.put(table, this.rocksdb.createColumnFamily(cfd)); ingestExternalFile(); }
/** * Creates a new column family with the name columnFamilyName and * allocates a ColumnFamilyHandle within an internal structure. * The ColumnFamilyHandle is automatically disposed with DB disposal. * * @param columnFamilyDescriptor column family to be created. * @return {@link org.rocksdb.ColumnFamilyHandle} instance. * * @throws RocksDBException thrown if error happens in underlying * native library. */ public ColumnFamilyHandle createColumnFamily( final ColumnFamilyDescriptor columnFamilyDescriptor) throws RocksDBException { return new ColumnFamilyHandle(this, createColumnFamily(nativeHandle_, columnFamilyDescriptor.columnFamilyName(), columnFamilyDescriptor.columnFamilyOptions().nativeHandle_)); }
private ColumnFamilyHandle getOrRegisterColumnFamilyHandle( ColumnFamilyDescriptor columnFamilyDescriptor, ColumnFamilyHandle columnFamilyHandle, StateMetaInfoSnapshot stateMetaInfoSnapshot) throws RocksDBException { Tuple2<ColumnFamilyHandle, RegisteredStateMetaInfoBase> registeredStateMetaInfoEntry = stateBackend.kvStateInformation.get(stateMetaInfoSnapshot.getName()); if (null == registeredStateMetaInfoEntry) { // create a meta info for the state on restore; // this allows us to retain the state in future snapshots even if it wasn't accessed RegisteredStateMetaInfoBase stateMetaInfo = RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot); registeredStateMetaInfoEntry = new Tuple2<>( columnFamilyHandle != null ? columnFamilyHandle : stateBackend.db.createColumnFamily(columnFamilyDescriptor), stateMetaInfo); stateBackend.registerKvStateInformation( stateMetaInfoSnapshot.getName(), registeredStateMetaInfoEntry); } return registeredStateMetaInfoEntry.f0; }
private ColumnFamilyHandle getOrRegisterColumnFamilyHandle( ColumnFamilyDescriptor columnFamilyDescriptor, ColumnFamilyHandle columnFamilyHandle, StateMetaInfoSnapshot stateMetaInfoSnapshot) throws RocksDBException { Tuple2<ColumnFamilyHandle, RegisteredStateMetaInfoBase> registeredStateMetaInfoEntry = stateBackend.kvStateInformation.get(stateMetaInfoSnapshot.getName()); if (null == registeredStateMetaInfoEntry) { // create a meta info for the state on restore; // this allows us to retain the state in future snapshots even if it wasn't accessed RegisteredStateMetaInfoBase stateMetaInfo = RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot); registeredStateMetaInfoEntry = new Tuple2<>( columnFamilyHandle != null ? columnFamilyHandle : stateBackend.db.createColumnFamily(columnFamilyDescriptor), stateMetaInfo); stateBackend.registerKvStateInformation( stateMetaInfoSnapshot.getName(), registeredStateMetaInfoEntry); } return registeredStateMetaInfoEntry.f0; }
@Override @VisibleForTesting public void deleteAllValues() throws IOException { exclusively((deferred) -> { synchronized(this) { deleteAllIterators(deferred); try { db.dropColumnFamily(handle); } catch(RocksDBException ex) { deferred.addException(ex); } deferred.suppressingClose(handle); try { this.handle = db.createColumnFamily(family); } catch (Exception ex) { deferred.addException(ex); } } }); }