/** * Creates and initializes a rocks block store. * * @param args inode store arguments */ public RocksInodeStore(InodeStoreArgs args) { mConf = args.getConf(); mBaseDir = mConf.get(PropertyKey.MASTER_METASTORE_DIR); RocksDB.loadLibrary(); mDisableWAL = new WriteOptions().setDisableWAL(true); mReadPrefixSameAsStart = new ReadOptions().setPrefixSameAsStart(true); try { initDb(); } catch (RocksDBException e) { throw new RuntimeException(e); } }
@Override public void put(String key, Object value) throws FailStoreException { String valueString = JSON.toJSONString(value); WriteOptions writeOpts = new WriteOptions(); try { writeOpts.setSync(true); writeOpts.setDisableWAL(true); db.put(writeOpts, key.getBytes("UTF-8"), valueString.getBytes("UTF-8")); } catch (Exception e) { throw new FailStoreException(e); } finally { writeOpts.dispose(); } }
@Override public void putBatch(Map<String, Object> map) { WriteOptions writeOpts = null; WriteBatch writeBatch = null; try { writeOpts = new WriteOptions(); writeBatch = new WriteBatch(); for (Entry<String, Object> entry : map.entrySet()) { String key = entry.getKey(); Object value = entry.getValue(); byte[] data = serialize(value); if (StringUtils.isBlank(key) || data == null || data.length == 0) { continue; } byte[] keyByte = key.getBytes(); writeBatch.put(keyByte, data); } db.write(writeOpts, writeBatch); } catch (Exception e) { LOG.error("Failed to putBatch into DB, " + map.keySet(), e); } finally { if (writeOpts != null) { writeOpts.dispose(); } if (writeBatch != null) { writeBatch.dispose(); } } }
@Override public boolean put(ObjectId commitId, ImmutableList<ObjectId> parentIds) { try (WriteBatchWithIndex batch = new WriteBatchWithIndex(); // RocksDBReference dbRef = dbhandle.getReference(); WriteOptions wo = new WriteOptions()) { wo.setSync(true); boolean updated = put(dbRef, commitId, parentIds, batch); dbRef.db().write(wo, batch); return updated; } catch (RocksDBException e) { throw new RuntimeException(e); } }
/** * Creates the RocksDB WriteOptions to use. Since we use RocksDB as an in-process cache with disk spillover, * we do not care about the data being persisted to disk for recovery purposes. As such: * * Write-Ahead-Log is disabled (2.8x performance improvement) * * Sync is disabled - does not wait for a disk flush before returning from the write call (50x or more improvement). */ private WriteOptions createWriteOptions() { return new WriteOptions() .setDisableWAL(true) .setSync(false); }
public void flush() throws RocksDBException { if (options != null) { db.write(options, batch); } else { // use the default WriteOptions, if wasn't provided. try (WriteOptions writeOptions = new WriteOptions()) { db.write(writeOptions, batch); } } batch.clear(); }
@Override public void flush() { try { for(Map.Entry<ByteArray, Map<ByteArray, byte[]>> entry: dataBatches.entrySet()) { WriteBatch writeBatch = new WriteBatch(); WriteOptions writeOptions = new WriteOptions().setDisableWAL(true); for(Map.Entry<ByteArray, byte[]> batchEntry: entry.getValue().entrySet()) { writeBatch.put( cfHandles.get(entry.getKey()), batchEntry.getKey().getBytes(), batchEntry.getValue() ); } rocksDB.write(writeOptions, writeBatch); writeBatch.close(); writeOptions.close(); entry.getValue().clear(); } FlushOptions fOptions = new FlushOptions().setWaitForFlush(true); rocksDB.flush(fOptions); fOptions.close(); } catch(RocksDBException ex) { throw new RuntimeException(ex); } }
private void putAll() { if (putBuffer.size() > 0) { Iterator<Entry<K, V>> iterator = putBuffer.entrySet().iterator(); try (WriteOptions wo = new WriteOptions()) { wo.setDisableWAL(true); wo.setSync(false); try (WriteBatch batch = new WriteBatch()) { while (iterator.hasNext()) { try (ByteArrayOutputStream keyBytes = new ByteArrayOutputStream(); ObjectOutput keyOut = new ObjectOutputStream(keyBytes); ByteArrayOutputStream valueBytes = new ByteArrayOutputStream(); ObjectOutput valueOut = new ObjectOutputStream(valueBytes);) { Entry<K, V> object = iterator.next(); keyOut.writeObject(object.getKey()); valueOut.writeObject(object.getValue()); keyOut.flush(); valueOut.flush(); batch.put(keyBytes.toByteArray(), valueBytes.toByteArray()); } } db.write(wo, batch); } wo.sync(); } catch (IOException | RocksDBException e) { throw Throwables.propagate(e); } putBuffer.clear(); } }
/** * If true, writes will not first go to the write ahead log, * and the write may got lost after a crash. * * @param flag a boolean flag to specify whether to disable * write-ahead-log on writes. * @return the instance of the current WriteOptions. */ public WriteOptions setDisableWAL(final boolean flag) { setDisableWAL(nativeHandle_, flag); return this; }
@Override public void close() throws IOException { writeBatch.close(); writeOptions.close(); }
/** * If true, the write will be flushed from the operating system * buffer cache (by calling WritableFile::Sync()) before the write * is considered complete. If this flag is true, writes will be * slower. * * If this flag is false, and the machine crashes, some recent * writes may be lost. Note that if it is just the process that * crashes (i.e., the machine does not reboot), no writes will be * lost even if sync==false. * * In other words, a DB write with sync==false has similar * crash semantics as the "write()" system call. A DB write * with sync==true has similar crash semantics to a "write()" * system call followed by "fdatasync()". * * Default: false * * @param flag a boolean flag to indicate whether a write * should be synchronized. * @return the instance of the current WriteOptions. */ public WriteOptions setSync(final boolean flag) { setSync(nativeHandle_, flag); return this; }
@Override public boolean put(ObjectId commitId, ImmutableList<ObjectId> parentIds) { try (WriteBatchWithIndex batch = new WriteBatchWithIndex(); // RocksDBReference dbRef = dbhandle.getReference(); WriteOptions wo = new WriteOptions()) { wo.setSync(true); boolean updated = put(dbRef, commitId, parentIds, batch); dbRef.db().write(wo, batch); return updated; } catch (Exception e) { throw Throwables.propagate(e); } }
/** * Builds RocksDb {@link WriteOptions}. * * @param sync * As of RocksDB v3.10.1, no data lost if process crashes even if sync==false. Data * lost only when server/OS crashes. So it's relatively safe to set sync=false for * fast write * @param disableWAL * @return */ public static WriteOptions buildWriteOptions(boolean sync, boolean disableWAL) { WriteOptions writeOptions = new WriteOptions(); writeOptions.setSync(sync).setDisableWAL(disableWAL); return writeOptions; }
@Override public void putBatch(Map<K, V> batch) throws IOException { try { WriteBatch writeBatch = new WriteBatch(); for (Map.Entry<K, V> entry : batch.entrySet()) { writeBatch.put( columnFamily, serializer.serialize(entry.getKey()), serializer.serialize(entry.getValue())); } rocksDb.write(new WriteOptions(), writeBatch); } catch (RocksDBException e) { throw new IOException(String.format("Failed to put batch=%s", batch), e); } }
@Override public void flush(String keySpace) { Preconditions.checkNotNull(keySpace); ByteArray byteArray = new ByteArray(keySpace); try { WriteBatch writeBatch = new WriteBatch(); WriteOptions writeOptions = new WriteOptions().setDisableWAL(true); for(Map.Entry<ByteArray, byte[]> entry: dataBatches.get(byteArray).entrySet()) { writeBatch.put( cfHandles.get(byteArray), entry.getKey().getBytes(), entry.getValue() ); } rocksDB.write(writeOptions, writeBatch); writeBatch.close(); writeOptions.close(); dataBatches.get(byteArray).clear(); FlushOptions fOptions = new FlushOptions().setWaitForFlush(true); rocksDB.flush(new FlushOptions(), cfHandles.get(byteArray)); fOptions.close(); } catch(RocksDBException ex) { throw new RuntimeException(ex); } }
writeOpts = new WriteOptions(); writeBatch = new WriteBatch(); } finally { if (writeOpts != null) { writeOpts.dispose();
public void close() { if (db != null) { writeOptions.close(); writeOptions = null; db.close(); db = null; deleteDB(); dbDir = null; } }
@Override public void removeConflicts(@Nullable String txId, Iterable<String> paths) { Optional<RocksDBReference> dbRefOpt = getDb(txId); if (!dbRefOpt.isPresent()) { return; } try (RocksDBReference dbRef = dbRefOpt.get(); WriteOptions writeOptions = new WriteOptions(); WriteBatch batch = new WriteBatch()) { writeOptions.setSync(true); for (String path : paths) { batch.remove(key(path)); } dbRef.db().write(writeOptions, batch); } catch (RocksDBException e) { throw new RuntimeException(e); } }
this.kvStateInformation = new LinkedHashMap<>(); this.writeOptions = new WriteOptions().setDisableWAL(true);
public RocksdbMap() { dbDir = Files.createTempDir(); try { this.db = RocksDB.open(dbDir.getAbsolutePath()); writeOptions = new WriteOptions(); writeOptions.setDisableWAL(true); writeOptions.setSync(false); } catch (RocksDBException e) { throw new RuntimeException(e); } }