public RocksDBLookupBuilder(TableDesc tableDesc, String[] keyColumns, String dbPath) { this.tableDesc = tableDesc; this.encoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns); this.dbPath = dbPath; this.writeBatchSize = 500; this.options = new Options(); options.setCreateIfMissing(true).setWriteBufferSize(8 * SizeUnit.KB).setMaxWriteBufferNumber(3) .setMaxBackgroundCompactions(5).setCompressionType(CompressionType.SNAPPY_COMPRESSION) .setCompactionStyle(CompactionStyle.UNIVERSAL); }
@Override public Options createOptions(Options currentOptions) { if (currentOptions == null) currentOptions = new Options(); currentOptions.setCreateIfMissing(true); currentOptions.setCreateMissingColumnFamilies(true); currentOptions.setMergeOperator(new StringAppendOperator()); currentOptions.setMaxWriteBufferNumber(4); currentOptions.setWriteBufferSize(64 * SizeUnit.MB); currentOptions.setTableFormatConfig(tableOptions); currentOptions.setTargetFileSizeBase(64 * SizeUnit.MB); currentOptions.setAllowOsBuffer(true); currentOptions.setMaxOpenFiles(-1); currentOptions.setMaxBackgroundFlushes(2); currentOptions.setMaxBackgroundCompactions(2); currentOptions.setCompactionStyle(CompactionStyle.LEVEL); currentOptions.setLevelZeroFileNumCompactionTrigger(4); currentOptions.setLevelZeroSlowdownWritesTrigger(20); currentOptions.setLevelZeroStopWritesTrigger(30); currentOptions.setNumLevels(4); currentOptions.setMaxBytesForLevelBase(64 * 4 * SizeUnit.MB); currentOptions.setAllowOsBuffer(false); return currentOptions;
public void initDb(List<Integer> list) throws Exception { Options dbOptions = new Options().setCreateMissingColumnFamilies(true).setCreateIfMissing(true); initDb(list, dbOptions); }
public static Options options(Config storeConfig, int numTasksForContainer) { Options options = new Options(); Long writeBufSize = storeConfig.getLong("container.write.buffer.size.bytes", 32 * 1024 * 1024); options.setWriteBufferSize((int) (writeBufSize / numTasksForContainer)); ", overwriting to " + compressionType.name()); options.setCompressionType(compressionType); BlockBasedTableConfig tableOptions = new BlockBasedTableConfig(); tableOptions.setBlockCacheSize(blockCacheSize).setBlockSize(blockSize); options.setTableFormatConfig(tableOptions); ", overwriting to " + compactionStyle.name()); options.setCompactionStyle(compactionStyle); options.setMaxWriteBufferNumber(storeConfig.getInt(ROCKSDB_NUM_WRITE_BUFFERS, 3)); options.setCreateIfMissing(true); options.setErrorIfExists(false); options.setMaxLogFileSize(storeConfig.getLong(ROCKSDB_MAX_LOG_FILE_SIZE_BYTES, 64 * 1024 * 1024L)); options.setKeepLogFileNum(storeConfig.getLong(ROCKSDB_KEEP_LOG_FILE_NUM, 2));
/** * Constructor. */ public RocksDBAtomicKVStore() { this.setOptions(new Options().setCreateIfMissing(true)); }
try (Options options = new Options()) { options.setCreateIfMissing(true); options.setCompressionType(CompressionType.LZ4_COMPRESSION); options.setWriteBufferSize(writeBufferSizeMB * 1024 * 1024); options.setMaxWriteBufferNumber(4); if (numLevels > 0) { options.setNumLevels(numLevels); options.setLevelZeroFileNumCompactionTrigger(numFilesInLevel0); options.setMaxBytesForLevelBase(maxSizeInLevel1MB * 1024 * 1024); options.setMaxBackgroundCompactions(16); options.setMaxBackgroundFlushes(16); options.setIncreaseParallelism(32); options.setMaxTotalWalSize(512 * 1024 * 1024); options.setMaxOpenFiles(-1); options.setTargetFileSizeBase(sstSizeMB * 1024 * 1024); options.setDeleteObsoleteFilesPeriodMicros(TimeUnit.HOURS.toMicros(1)); options.setLevelCompactionDynamicLevelBytes(true); options.setTableFormatConfig(tableOptions); switch (logLevel) { case "debug": options.setInfoLogLevel(InfoLogLevel.DEBUG_LEVEL); break; case "info":
env.setBackgroundThreads(8, Env.FLUSH_POOL); env.setBackgroundThreads(8, Env.COMPACTION_POOL); Options options = new Options(); options.setCreateIfMissing(true); options.setCompactionStyle(CompactionStyle.LEVEL); options.setCompressionType(CompressionType.NO_COMPRESSION); options.setLevel0FileNumCompactionTrigger(8); options.setMaxBackgroundCompactions(2); options.setMaxBackgroundFlushes(8); options.setEnv(env); options.setAccessHintOnCompactionStart(AccessHint.WILLNEED); options.setIncreaseParallelism(32); options.setAdviseRandomOnOpen(true); options.setAllowConcurrentMemtableWrite(true); options.setWriteBufferSize(bufferSize); options.setCompactionReadaheadSize(1024 * 1024 * 25); options.setStatsDumpPeriodSec(30); options.setMaxOpenFiles(-1); options.setMaxBytesForLevelBase(fsize * 5); options.setTargetFileSizeBase(fsize); options.setTableFormatConfig(blockConfig); File f = new File(fileName + File.separator); f.mkdirs();
tableConfig.setBlockSize(BLOCK_SIZE); options = new Options(); options.setTableFormatConfig(tableConfig); options.setWriteBufferSize(WRITE_BUFFER_SIZE); options.setCompressionType(COMPRESSION_TYPE); options.setCompactionStyle(COMPACTION_STYLE); options.setMaxWriteBufferNumber(MAX_WRITE_BUFFERS); options.setCreateIfMissing(true); options.setErrorIfExists(false); options.setInfoLogLevel(InfoLogLevel.ERROR_LEVEL); options.setIncreaseParallelism(Math.max(Runtime.getRuntime().availableProcessors(), 2)); options.prepareForBulkLoad();
public RocksDbCacheOperator(TopologyContext context, String cacheDir) { this.stormConf = context.getStormConf(); this.maxFlushSize = ConfigExtension.getTransactionCacheBatchFlushSize(stormConf); Options rocksDbOpt = new Options(); rocksDbOpt.setCreateMissingColumnFamilies(true).setCreateIfMissing(true); long bufferSize = ConfigExtension.getTransactionCacheBlockSize(stormConf) != null ? ConfigExtension.getTransactionCacheBlockSize(stormConf) : (1 * SizeUnit.GB); rocksDbOpt.setWriteBufferSize(bufferSize); int maxBufferNum = ConfigExtension.getTransactionMaxCacheBlockNum(stormConf) != null ? ConfigExtension.getTransactionMaxCacheBlockNum(stormConf) : 3; rocksDbOpt.setMaxWriteBufferNumber(maxBufferNum); // Config for log of RocksDb rocksDbOpt.setMaxLogFileSize(1073741824); // 1G rocksDbOpt.setKeepLogFileNum(1); rocksDbOpt.setInfoLogLevel(InfoLogLevel.WARN_LEVEL); try { Map<Object, Object> conf = new HashMap<Object, Object>(); conf.put(ROCKSDB_ROOT_DIR, cacheDir); conf.put(ROCKSDB_RESET, true); initDir(conf); initDb(null, rocksDbOpt); } catch (Exception e) { throw new RuntimeException(e); } kryo = new Kryo(); output = new Output(200, 2000000000); input = new Input(1); LOG.info("Finished rocksDb cache init: maxFlushSize={}, bufferSize={}, maxBufferNum={}", maxFlushSize, bufferSize, maxBufferNum); }
@Override public SuffixDataStore createSuffixDataStore(int id) { RocksDB.loadLibrary(); try (Options options = new Options().setCreateIfMissing(true).prepareForBulkLoad()) { Files.createDirectories(dbPath); RocksDB rocksDB = RocksDB.open(options, dbPath.resolve(getSuffixesName(id)).toString()); rocksDBS.add(rocksDB); return new RocksDbSuffixDataStore(rocksDB); } catch (RocksDBException | IOException e) { throw new RuntimeException(e); } }
.setNumLevels(4) .setMaxWriteBufferNumber(maxWriteBufferNumber); rocksDBOptions = new Options(dbOptions, cfOptions) .setCreateIfMissing(true) .setCreateMissingColumnFamilies(true) .optimizeLevelStyleCompaction(memtableSize) .setCompactionStyle(CompactionStyle.LEVEL) .setNumLevels(4) .setIncreaseParallelism(parallelism) .setCompactionReadaheadSize(compactionReadAheadSize) .setMaxBackgroundCompactions(maxBackgroundCompactions) .setMaxBackgroundFlushes(maxBackgroundFlushes) .setMaxWriteBufferNumber(maxWriteBufferNumber) .setWalSizeLimitMB(0L) .setWalTtlSeconds(0L); rocksDBOptions.setMaxSubcompactions(maxSubcompactions);
Options options = new Options(); options.setCreateMissingColumnFamilies(true); options.setCreateIfMissing(true); options.setTargetFileSizeBase(fileSizeBase); options.setMaxBackgroundFlushes(2); options.setMaxBackgroundCompactions(2); options.setCompactionStyle(CompactionStyle.LEVEL); options.setNumLevels(levelNum); options.setLevelZeroFileNumCompactionTrigger(compactionTriggerNum);
final boolean safeToUseMMappedFiles = !isWindows; Options options = new Options(); options.setCreateIfMissing(true)// .setAdviseRandomOnOpen(true)// .setAllowMmapReads(safeToUseMMappedFiles)// .setAllowMmapWrites(safeToUseMMappedFiles)// .setWriteBufferSize(16 * 1024 * 1024)// .setMaxWriteBufferNumber(8)// .setMaxBackgroundCompactions(2)// .setMinWriteBufferNumberToMerge(4)// .setBytesPerSync(0/* 64 * 1024 * 1024 */)// .setCompressionType(CompressionType.NO_COMPRESSION);
try (Options options = new Options()) { options.setCreateIfMissing(true); options.setCompressionType(CompressionType.LZ4_COMPRESSION); options.setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION); options.setLevelCompactionDynamicLevelBytes(true); options.setMaxOpenFiles(settings.getMaxOpenFiles()); options.setIncreaseParallelism(settings.getMaxThreads()); options.useFixedLengthPrefixExtractor(NodeKeyCompositor.PREFIX_BYTES); options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig()); tableCfg.setBlockSize(16 * 1024); tableCfg.setBlockCacheSize(32 * 1024 * 1024);
private Options createDatabaseOptions() { BlockBasedTableConfig tableFormatConfig = new BlockBasedTableConfig() .setBlockSize(cacheBlockSizeKB * 1024L) .setBlockCacheSize(readCacheSizeMB * 1024L * 1024L) .setCacheIndexAndFilterBlocks(true); return new Options() .setCreateIfMissing(true) .setDbLogDir(Paths.get(this.dbDir, DB_LOG_DIR).toString()) .setWalDir(Paths.get(this.dbDir, DB_WRITE_AHEAD_LOG_DIR).toString()) .setWalTtlSeconds(0) .setWalSizeLimitMB(MAX_WRITE_AHEAD_LOG_SIZE_MB) .setWriteBufferSize(writeBufferSizeMB * 1024L * 1024L) .setMaxWriteBufferNumber(MAX_WRITE_BUFFER_NUMBER) .setMinWriteBufferNumberToMerge(MIN_WRITE_BUFFER_NUMBER_TO_MERGE) .setTableFormatConfig(tableFormatConfig) .setOptimizeFiltersForHits(true) .setUseDirectReads(true); }
final Options options = new Options() .optimizeLevelStyleCompaction() .setCreateIfMissing(true) .setCreateMissingColumnFamilies(true) .setIncreaseParallelism(rocksThreads) .setMaxBackgroundCompactions(rocksThreads) .setInfoLogLevel(InfoLogLevel.INFO_LEVEL); dbOptions = options; return RocksDB.open(options, rocksDbDir.toAbsolutePath().toString());
public RocksDBLookupTable(TableDesc tableDesc, String[] keyColumns, String dbPath) { this.options = new Options(); this.rowEncoder = new RocksDBLookupRowEncoder(tableDesc, keyColumns); try { this.rocksDB = RocksDB.openReadOnly(options, dbPath); } catch (RocksDBException e) { throw new IllegalStateException("cannot open rocks db in path:" + dbPath, e); } }
@Override public void setup(final BenchmarkParams b) throws IOException { super.setup(b); wkb = new UnsafeBuffer(new byte[keySize]); wvb = new UnsafeBuffer(new byte[valSize]); loadLibrary(); final Options options = new Options(); options.setCreateIfMissing(true); options.setCompressionType(NO_COMPRESSION); try { db = open(options, tmp.getAbsolutePath()); } catch (final RocksDBException ex) { throw new IOException(ex); } }
public synchronized RocksDBDataIndexTable getDataIndexTable( final String tableName, final short adapterId) { if (indexWriteOptions == null) { RocksDB.loadLibrary(); final int cores = Runtime.getRuntime().availableProcessors(); indexWriteOptions = new Options().setCreateIfMissing(true).prepareForBulkLoad().setIncreaseParallelism(cores); indexReadOptions = new Options().setIncreaseParallelism(cores); } final String directory = subDirectory + "/" + tableName; return dataIndexTableCache.get( (DataIndexCacheKey) keyCache.get(directory, d -> new DataIndexCacheKey(d, adapterId))); }
/** * Builds RocksDb {@link Options}. * * @param maxBackgroundThreads * @param levelZeloFileNumCompactionTrigger * @param writeBufferSize * @param targetFileSizeBase * @return */ public static Options buildOptions(int maxBackgroundThreads, int levelZeloFileNumCompactionTrigger, long writeBufferSize, long targetFileSizeBase) { Options rocksOptions = new Options(); rocksOptions.setCreateIfMissing(true).getEnv().setBackgroundThreads(1, Env.FLUSH_POOL) .setBackgroundThreads(maxBackgroundThreads, Env.COMPACTION_POOL); rocksOptions.setMaxBackgroundFlushes(1).setMaxBackgroundCompactions(maxBackgroundThreads); rocksOptions.setWriteBufferSize(writeBufferSize).setMinWriteBufferNumberToMerge(2) .setLevelZeroFileNumCompactionTrigger(levelZeloFileNumCompactionTrigger) .setTargetFileSizeBase(targetFileSizeBase); rocksOptions.setMemTableConfig(new SkipListMemTableConfig()); // rocksOptions.setMemTableConfig(new HashSkipListMemTableConfig()); // rocksOptions.setMemTableConfig(new HashLinkedListMemTableConfig()); return rocksOptions; }