/** * This method recreates and registers all {@link ColumnFamilyDescriptor} from Flink's state meta data snapshot. */ private List<ColumnFamilyDescriptor> createAndRegisterColumnFamilyDescriptors( List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) { List<ColumnFamilyDescriptor> columnFamilyDescriptors = new ArrayList<>(stateMetaInfoSnapshots.size()); for (StateMetaInfoSnapshot stateMetaInfoSnapshot : stateMetaInfoSnapshots) { ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor( stateMetaInfoSnapshot.getName().getBytes(ConfigConstants.DEFAULT_CHARSET), stateBackend.columnOptions); columnFamilyDescriptors.add(columnFamilyDescriptor); } return columnFamilyDescriptors; }
public static RocksDB createWithColumnFamily(Map conf, String rocksDbDir, final Map<String, ColumnFamilyHandle> columnFamilyHandleMap, int ttlTimeSec) throws IOException { List<ColumnFamilyDescriptor> columnFamilyDescriptors = getExistingColumnFamilyDesc(conf, rocksDbDir); List<ColumnFamilyHandle> columnFamilyHandles = new ArrayList<>(); DBOptions dbOptions = getDBOptions(conf); try { RocksDB rocksDb = ttlTimeSec > 0 ? TtlDB.open( dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles, getTtlValues(ttlTimeSec, columnFamilyDescriptors), false) : RocksDB.open(dbOptions, rocksDbDir, columnFamilyDescriptors, columnFamilyHandles); int n = Math.min(columnFamilyDescriptors.size(), columnFamilyHandles.size()); // skip default column columnFamilyHandleMap.put(DEFAULT_COLUMN_FAMILY, rocksDb.getDefaultColumnFamily()); for (int i = 1; i < n; i++) { ColumnFamilyDescriptor descriptor = columnFamilyDescriptors.get(i); columnFamilyHandleMap.put(new String(descriptor.columnFamilyName()), columnFamilyHandles.get(i)); } LOG.info("Finished loading RocksDB with existing column family={}, dbPath={}, ttlSec={}", columnFamilyHandleMap.keySet(), rocksDbDir, ttlTimeSec); // enable compaction rocksDb.compactRange(); return rocksDb; } catch (RocksDBException e) { throw new IOException("Failed to initialize RocksDb.", e); } }
@Override public void createTable(String table) throws RocksDBException { if (this.cfs.containsKey(table)) { return; } this.checkValid(); // Should we use options.setCreateMissingColumnFamilies() to create CF ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(table)); ColumnFamilyOptions options = cfd.columnFamilyOptions(); initOptions(this.conf, null, options, options); this.cfs.put(table, this.rocksdb.createColumnFamily(cfd)); ingestExternalFile(); }
protected void prepareColumnFamilyDescriptors() throws RocksDbException { if (columnFamilies == null || columnFamilies.size() == 0) { if (columnFamilies == null) { columnFamilies = new HashSet<>(); } try { String[] cfList = RocksDbUtils.getColumnFamilyList(directory.getAbsolutePath()); for (String cf : cfList) { columnFamilies.add(RocksDbUtils.buildColumnFamilyDescriptor(cf)); } } catch (Exception e) { throw e instanceof RocksDbException ? (RocksDbException) e : new RocksDbException(e); } } boolean hasDefaultCf = false; for (ColumnFamilyDescriptor cfd : columnFamilies) { if (Arrays.equals(cfd.columnFamilyName(), RocksDB.DEFAULT_COLUMN_FAMILY)) { hasDefaultCf = true; break; } } if (!hasDefaultCf) { columnFamilies.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY)); } }
/** * Creates a new column family with the name columnFamilyName and * allocates a ColumnFamilyHandle within an internal structure. * The ColumnFamilyHandle is automatically disposed with DB disposal. * * @param columnFamilyDescriptor column family to be created. * @return {@link org.rocksdb.ColumnFamilyHandle} instance. * * @throws RocksDBException thrown if error happens in underlying * native library. */ public ColumnFamilyHandle createColumnFamily( final ColumnFamilyDescriptor columnFamilyDescriptor) throws RocksDBException { return new ColumnFamilyHandle(this, createColumnFamily(nativeHandle_, columnFamilyDescriptor.columnFamilyName(), columnFamilyDescriptor.columnFamilyOptions().nativeHandle_)); }
public static <ColumnFamilyNames extends Enum<ColumnFamilyNames>> ZeebeRocksDb<ColumnFamilyNames> openZbDb( final DBOptions options, final String path, final List<ColumnFamilyDescriptor> columnFamilyDescriptors, final List<AutoCloseable> closables, Class<ColumnFamilyNames> columnFamilyTypeClass) throws RocksDBException { final EnumMap<ColumnFamilyNames, Long> columnFamilyMap = new EnumMap<>(columnFamilyTypeClass); final byte[][] cfNames = new byte[columnFamilyDescriptors.size()][]; final long[] cfOptionHandles = new long[columnFamilyDescriptors.size()]; for (int i = 0; i < columnFamilyDescriptors.size(); i++) { final ColumnFamilyDescriptor cfDescriptor = columnFamilyDescriptors.get(i); cfNames[i] = cfDescriptor.getName(); cfOptionHandles[i] = getNativeHandle(cfDescriptor.getOptions()); } final long[] handles = open(getNativeHandle(options), path, cfNames, cfOptionHandles); final ColumnFamilyNames[] enumConstants = columnFamilyTypeClass.getEnumConstants(); for (int i = 1; i < handles.length; i++) { columnFamilyMap.put(enumConstants[i - 1], handles[i]); } final ZeebeRocksDb<ColumnFamilyNames> db = new ZeebeRocksDb<ColumnFamilyNames>( handles[0], columnFamilyMap, closables, columnFamilyTypeClass); db.storeOptionsInstance(options); return db; }
/** * Retrieve name of column family. * * @return column family name. * @since 3.10.0 * * @deprecated Use {@link #getName()} instead. */ @Deprecated public byte[] columnFamilyName() { return getName(); }
ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(cf)); ColumnFamilyOptions options = cfd.columnFamilyOptions(); RocksDBStdSessions.initOptions(this.conf, null, options, options); cfds.add(cfd);
byte[] colFamilyName = name.getBytes(Charsets.UTF_8); ColumnFamilyOptions colFamilyOptions = newColFamilyOptions(); colDescriptors.add(new ColumnFamilyDescriptor(colFamilyName, colFamilyOptions)); int index = 0; for (ColumnFamilyDescriptor descriptor : colDescriptors) { if (new String(descriptor.columnFamilyName()).equals("default")) { extraColumns.put("default", colFamiliesTarget.get(index));
/** * <p>Creates a new ttl based column family with a name defined * in given ColumnFamilyDescriptor and allocates a * ColumnFamilyHandle within an internal structure.</p> * * <p>The ColumnFamilyHandle is automatically disposed with DB * disposal.</p> * * @param columnFamilyDescriptor column family to be created. * @param ttl TTL to set for this column family. * * @return {@link org.rocksdb.ColumnFamilyHandle} instance. * * @throws RocksDBException thrown if error happens in underlying * native library. */ public ColumnFamilyHandle createColumnFamilyWithTtl( final ColumnFamilyDescriptor columnFamilyDescriptor, final int ttl) throws RocksDBException { return new ColumnFamilyHandle(this, createColumnFamilyWithTtl(nativeHandle_, columnFamilyDescriptor.columnFamilyName(), columnFamilyDescriptor.columnFamilyOptions().nativeHandle_, ttl)); }
private synchronized ColumnFamilyHandle getOrCreateColumnFamily(String kvStoreId) throws IOException { ColumnFamilyHandle columnFamily = columnFamilies.get(kvStoreId); if (columnFamily == null) { ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(kvStoreId.getBytes(), columnOptions); try { columnFamily = rocksDB.createColumnFamily(columnDescriptor); columnFamilies.put(kvStoreId, columnFamily); } catch (RocksDBException e) { throw new IOException("Error creating ColumnFamilyHandle.", e); } } return columnFamily; }
@Override public void createTable(String table) throws RocksDBException { if (this.cfs.containsKey(table)) { return; } this.checkValid(); // Should we use options.setCreateMissingColumnFamilies() to create CF ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(table)); ColumnFamilyOptions options = cfd.columnFamilyOptions(); initOptions(this.conf, null, options, options); this.cfs.put(table, this.rocksdb.createColumnFamily(cfd)); ingestExternalFile(); }
byte[] colFamilyName = name.getBytes(Charsets.UTF_8); ColumnFamilyOptions colFamilyOptions = newColFamilyOptions(); colDescriptors.add(new ColumnFamilyDescriptor(colFamilyName, colFamilyOptions)); int index = 0; for (ColumnFamilyDescriptor descriptor : colDescriptors) { if (new String(descriptor.columnFamilyName()).equals("default")) { extraColumns.put("default", colFamiliesTarget.get( index));
final ColumnFamilyDescriptor cfDescriptor = columnFamilyDescriptors.get(i); cfNames[i] = cfDescriptor.columnFamilyName(); cfOptionHandles[i] = cfDescriptor.columnFamilyOptions().nativeHandle_;
synchronized public RocksDbWrapper setColumnFamilies( Collection<ColumnFamilyDescriptor> columnFamilies) { this.columnFamilies.clear(); this.columnFamilyNames.clear(); for (Entry<String, ColumnFamilyHandle> entry : this.columnFamilyHandles.entrySet()) { RocksDbUtils.closeRocksObjects(entry.getValue()); } if (columnFamilies != null) { this.columnFamilies.addAll(columnFamilies); this.columnFamilies.iterator().forEachRemaining(x -> { this.columnFamilyNames.add(new String(x.columnFamilyName(), RocksDbUtils.UTF8)); }); } return this; }
private ColumnFamilyHandle getOrCreateColumnFamily(String kvStoreId) throws IOException { ColumnFamilyHandle columnFamily = columnFamilies.get(kvStoreId); // TODO: verify existing column family is compatible with the request. if (columnFamily == null) { ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(kvStoreId.getBytes(), columnOptions); try { columnFamily = rocksDB.createColumnFamily(columnDescriptor); columnFamilies.put(kvStoreId, columnFamily); } catch (RocksDBException e) { throw new IOException("Error creating ColumnFamilyHandle.", e); } } return columnFamily; }
ColumnFamilyDescriptor cfd = new ColumnFamilyDescriptor(encode(cf)); ColumnFamilyOptions options = cfd.columnFamilyOptions(); RocksDBStdSessions.initOptions(this.conf, null, options, options); cfds.add(cfd);
final ColumnFamilyDescriptor cfDescriptor = columnFamilyDescriptors .get(i); cfNames[i] = cfDescriptor.columnFamilyName(); cfOptionHandles[i] = cfDescriptor.columnFamilyOptions().nativeHandle_;
public void initialize(List<ColumnFamilyDescriptor> cfDescriptors, List<ColumnFamilyHandle> cfHandles) throws RocksDBException { for (int i = 0; i < cfDescriptors.size(); i++) { String fullTableName = new String(cfDescriptors.get(i).columnFamilyName(), Charsets.UTF_8); int nameIndex = fullTableName.lastIndexOf("__"); String tableName; long index; if (nameIndex == -1) { tableName = fullTableName; index = 0; } else { tableName = fullTableName.substring(0, nameIndex); index = Long.parseLong(fullTableName.substring(nameIndex + 2)); } ColumnFamily cf = new ColumnFamily(index, cfHandles.get(i)); ColumnFamily oldCf = cfs.put(tableName, cf); if (oldCf != null && !tableName.equals("default")) { db.dropColumnFamily(oldCf.getHandle()); oldCf.getHandle().dispose(); } } }
/** * Creates a column family handle for use with a k/v state. */ private ColumnFamilyHandle createColumnFamily(String stateName) { byte[] nameBytes = stateName.getBytes(ConfigConstants.DEFAULT_CHARSET); Preconditions.checkState(!Arrays.equals(RocksDB.DEFAULT_COLUMN_FAMILY, nameBytes), "The chosen state name 'default' collides with the name of the default column family!"); ColumnFamilyDescriptor columnDescriptor = new ColumnFamilyDescriptor(nameBytes, columnOptions); try { return db.createColumnFamily(columnDescriptor); } catch (RocksDBException e) { throw new FlinkRuntimeException("Error creating ColumnFamilyHandle.", e); } }