public RocksDBLookupTable(TableDesc tableDesc, String[] keyColumns, String dbPath) { this.options = new Options(); this.rowEncoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns); try { this.rocksDB = RocksDB.openReadOnly(options, dbPath); } catch (RocksDBException e) { throw new IllegalStateException("cannot open rocks db in path:" + dbPath, e); } }
public void connect() throws IOException { try { if (Files.exists(fastaFilePath) && Files.isDirectory(fastaFilePath)) { rocksDB = RocksDB.openReadOnly(fastaFilePath.toAbsolutePath().toString()); connected = true; } else { Path rocksdbIdxPath = Paths.get(fastaFilePath.toString() + INDEX_EXTENSION); if (Files.exists(rocksdbIdxPath) && Files.isDirectory(rocksdbIdxPath)) { rocksDB = RocksDB.openReadOnly(rocksdbIdxPath.toString()); connected = true; } } } catch (RocksDBException e) { throw new IOException("Error while connecting", e); } }
@SuppressFBWarnings( justification = "double check for null is intentional to avoid synchronized blocks when not needed.") protected RocksDB getReadDb() { if (!exists) { return null; } // avoid synchronization if unnecessary by checking for null outside // synchronized block if (readDb == null) { synchronized (this) { // check again within synchronized block if (readDb == null) { try { readerDirty = false; readDb = RocksDB.openReadOnly(readOptions, subDirectory); } catch (final RocksDBException e) { LOGGER.warn("Unable to open for reading", e); } } } } return readDb; } }
public RocksDbStrings(Path termsPath) { RocksDB.loadLibrary(); try { terms = RocksDB.openReadOnly(termsPath.toString()); } catch (RocksDBException e) { // says "if error happens in underlying native library", can't possible hope to handle that. throw new RuntimeException(e); } }
/** * The factory constructor of RocksDB that opens a RocksDB instance in * Read-Only mode given the path to the database using the default * options. * * @param path the path to the RocksDB. * @return a {@link RocksDB} instance on success, null if the specified * {@link RocksDB} can not be opened. * * @throws RocksDBException thrown if error happens in underlying * native library. */ public static RocksDB openReadOnly(final String path) throws RocksDBException { // This allows to use the rocksjni default Options instead of // the c++ one. Options options = new Options(); return openReadOnly(options, path); }
public RocksDBLookupTable(TableDesc tableDesc, String[] keyColumns, String dbPath) { this.options = new Options(); this.rowEncoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns); try { this.rocksDB = RocksDB.openReadOnly(options, dbPath); } catch (RocksDBException e) { throw new IllegalStateException("cannot open rocks db in path:" + dbPath, e); } }
/** * The factory constructor of RocksDB that opens a RocksDB instance in * Read-Only mode given the path to the database using the default * options. * * @param path the path to the RocksDB. * @param columnFamilyDescriptors list of column family descriptors * @param columnFamilyHandles will be filled with ColumnFamilyHandle instances * on open. * @return a {@link RocksDB} instance on success, null if the specified * {@link RocksDB} can not be opened. * * @throws RocksDBException thrown if error happens in underlying * native library. */ public static RocksDB openReadOnly(final String path, final List<ColumnFamilyDescriptor> columnFamilyDescriptors, final List<ColumnFamilyHandle> columnFamilyHandles) throws RocksDBException { // This allows to use the rocksjni default Options instead of // the c++ one. final DBOptions options = new DBOptions(); return openReadOnly(options, path, columnFamilyDescriptors, columnFamilyHandles); }
RocksDBNormalizerModel(Path dbPath) { RocksDB.loadLibrary(); try (Options options = new Options().setInfoLogLevel(InfoLogLevel.ERROR_LEVEL)) { db = RocksDB.openReadOnly(options, dbPath.toString()); } catch (RocksDBException e) { throw new RuntimeException(e); } }
public RocksDbIdentifiers(Path identifiersPath) { RocksDB.loadLibrary(); try (Options options = new Options().setInfoLogLevel(InfoLogLevel.ERROR_LEVEL)) { indices = RocksDB.openReadOnly(options, identifiersPath.toString()); } catch (RocksDBException e) { throw new RuntimeException(e); } }
@Override public SuffixDataStore openSuffixDataStore(int id) { RocksDB.loadLibrary(); try { LOGGER.info("Opening TnT suffix model: {}", id); RocksDB rocksDB = RocksDB.openReadOnly(dbPath.resolve(getSuffixesName(id)).toString()); RocksDbSuffixDataStore rocksDbSuffixDataStore = new RocksDbSuffixDataStore(rocksDB); if (inMemory) { LOGGER.info("Loading TnT suffix model into memory: {}", id); InMemorySuffixDataStore inMemorySuffixDataStore = rocksDbSuffixDataStore.inMemory(); LOGGER.info("Done loading TnT suffix model into memory: {}", id); rocksDB.close(); return inMemorySuffixDataStore; } rocksDBS.add(rocksDB); return rocksDbSuffixDataStore; } catch (RocksDBException e) { throw new RuntimeException(e); } }
@Override public KnownWordsDataStore openKnownWordDataStore(int id) { RocksDB.loadLibrary(); try { LOGGER.info("Opening TnT model known word model: {}", id); RocksDB rocksDB = RocksDB.openReadOnly(dbPath.resolve(getWordsName(id)).toString()); RocksDB candidatesDB = RocksDB.openReadOnly(dbPath.resolve(getCandidatesName(id)).toString()); RocksDbKnownWordsDataStore rocksDbKnownWordsDataStore = new RocksDbKnownWordsDataStore( rocksDB, candidatesDB); if (inMemory) { LOGGER.info("Loading TnT known word model into memory: {}", id); InMemoryKnownWordDataStore inMemoryKnownWordDataStore = rocksDbKnownWordsDataStore .inMemory(); LOGGER.info("Done loading TnT known word model into memory: {}", id); rocksDB.close(); candidatesDB.close(); return inMemoryKnownWordDataStore; } rocksDBS.add(rocksDB); rocksDBS.add(candidatesDB); return rocksDbKnownWordsDataStore; } catch (RocksDBException e) { throw new RuntimeException(e); } }
public RocksDBSenseVectors(Path path, boolean forWriting) { RocksDB.loadLibrary(); if (forWriting) { try (Options options = new Options().setCreateIfMissing(true).prepareForBulkLoad()) { rocksDB = RocksDB.open(options, path.toString()); } catch (RocksDBException e) { throw new RuntimeException(e); } } else { try { rocksDB = RocksDB.openReadOnly(path.toString()); } catch (RocksDBException e) { throw new RuntimeException(e); } } }
LOGGER.info("Opening concepts dictionary: {}. inMemory = {}.", dbPath, inMemory); RocksDB phrasesDB = RocksDB.openReadOnly(options, dbPath.resolve("phrases").toString()); RocksDB lowercaseDB = RocksDB.openReadOnly(options, dbPath.resolve("lowercase").toString()); RocksDB normsDB = RocksDB.openReadOnly(options, dbPath.resolve("norms").toString()); Map<Integer, String> sources = new HashMap<>();
myOwnReadOptions = false; rocksDb = RocksDB.openReadOnly(dbOptions, path, cfdList, cfhList); } else { if (writeOptions == null) {
private Object[] getDBConnection(String dbLocation, boolean forceCreate) { boolean indexingNeeded = forceCreate || !Files.exists(Paths.get(dbLocation)); // a static method that loads the RocksDB C++ library. RocksDB.loadLibrary(); // the Options class contains a set of configurable DB options // that determines the behavior of a database. Options options = new Options().setCreateIfMissing(true); RocksDB db = null; try { // a factory method that returns a RocksDB instance if (indexingNeeded) { db = RocksDB.open(options, dbLocation); } else { db = RocksDB.openReadOnly(options, dbLocation); } // do something } catch (RocksDBException e) { // do some error handling e.printStackTrace(); System.exit(1); } return new Object[]{db, options, dbLocation, indexingNeeded}; }
private Object[] getDBConnection(String dbLocation, boolean forceCreate) { boolean indexingNeeded = forceCreate || !Files.exists(Paths.get(dbLocation)); // a static method that loads the RocksDB C++ library. RocksDB.loadLibrary(); // the Options class contains a set of configurable DB options // that determines the behavior of a database. Options options = new Options().setCreateIfMissing(true); if (maxOpenFiles > 0) { options.setMaxOpenFiles(maxOpenFiles); } RocksDB db = null; try { // a factory method that returns a RocksDB instance if (indexingNeeded) { db = RocksDB.open(options, dbLocation); } else { db = RocksDB.openReadOnly(options, dbLocation); } // do something } catch (RocksDBException e) { // do some error handling e.printStackTrace(); System.exit(1); } return new Object[]{db, options, dbLocation, indexingNeeded}; }
public RocksDBReader(StoreProperties props) { props.required(LOCATION); String location = props.get(LOCATION) + props.get(CACHE_NAME).replaceAll("[^a-zA-Z0-9-_\\.]", "_"); File f = new File(location); if (!f.exists() || !f.isDirectory()) throw new CacheException(String.format("Unable to read db directory '%s'", location)); Options options = new Options().setCreateIfMissing(false); String compressionType = props.get(COMPRESSION); if (compressionType != null) { options.setCompressionType(CompressionType.getCompressionType(compressionType)); } try { this.db = RocksDB.openReadOnly(options, location); } catch (RocksDBException e) { throw new CacheException(e); } this.marshaller = SerializationConfigUtil.getMarshaller(props); }
/** * Construct the <code>RocksDbKeyValueReader</code> with store's name, * database's path and Samza's config * * @param storeName name of the RocksDb defined in the config file * @param dbPath path to the db directory * @param config Samza's config */ public RocksDbKeyValueReader(String storeName, String dbPath, Config config) { // get the key serde and value serde from the config JavaStorageConfig storageConfig = new JavaStorageConfig(config); JavaSerializerConfig serializerConfig = new JavaSerializerConfig(config); keySerde = getSerdeFromName(storageConfig.getStorageKeySerde(storeName), serializerConfig); valueSerde = getSerdeFromName(storageConfig.getStorageMsgSerde(storeName), serializerConfig); // get db options Options options = RocksDbOptionsHelper.options(config, 1, new File(dbPath), StorageEngineFactory.StoreMode.ReadWrite); // open the db RocksDB.loadLibrary(); try { db = RocksDB.openReadOnly(options, dbPath); } catch (RocksDBException e) { throw new SamzaException("can not open the rocksDb in " + dbPath, e); } }
/** * Construct the <code>RocksDbKeyValueReader</code> with store's name, * database's path and Samza's config * * @param storeName name of the RocksDb defined in the config file * @param dbPath path to the db directory * @param config Samza's config */ public RocksDbKeyValueReader(String storeName, String dbPath, Config config) { // get the key serde and value serde from the config JavaStorageConfig storageConfig = new JavaStorageConfig(config); JavaSerializerConfig serializerConfig = new JavaSerializerConfig(config); keySerde = getSerdeFromName(storageConfig.getStorageKeySerde(storeName), serializerConfig); valueSerde = getSerdeFromName(storageConfig.getStorageMsgSerde(storeName), serializerConfig); // get db options Options options = RocksDbOptionsHelper.options(config, 1); // open the db RocksDB.loadLibrary(); try { db = RocksDB.openReadOnly(options, dbPath); } catch (RocksDBException e) { throw new SamzaException("can not open the rocksDb in " + dbPath, e); } }
/** * Construct the <code>RocksDbKeyValueReader</code> with store's name, * database's path and Samza's config * * @param storeName name of the RocksDb defined in the config file * @param dbPath path to the db directory * @param config Samza's config */ public RocksDbKeyValueReader(String storeName, String dbPath, Config config) { // get the key serde and value serde from the config JavaStorageConfig storageConfig = new JavaStorageConfig(config); JavaSerializerConfig serializerConfig = new JavaSerializerConfig(config); keySerde = getSerdeFromName(storageConfig.getStorageKeySerde(storeName), serializerConfig); valueSerde = getSerdeFromName(storageConfig.getStorageMsgSerde(storeName), serializerConfig); // get db options Options options = RocksDbOptionsHelper.options(config, 1); // open the db RocksDB.loadLibrary(); try { db = RocksDB.openReadOnly(options, dbPath); } catch (RocksDBException e) { throw new SamzaException("can not open the rocksDb in " + dbPath, e); } }