public byte[] value() { return iterator.value(); }
private static int getStartValue(RocksDB db, ColumnFamilyHandle handler) { int startValue = 0; SortedMap<Integer, Integer> values = new TreeMap<Integer, Integer>(); RocksIterator itr = db.newIterator(handler); itr.seekToFirst(); while(itr.isValid()) { Integer key = Integer.valueOf(new String(itr.key())); Integer value = Integer.valueOf(new String(itr.value())); values.put(key, value); itr.next(); } LOG.info("Load previous db: size={}, values={}", values.size(), values); if (!values.isEmpty()) { Integer key = values.lastKey(); startValue = values.get(key); startValue++; } LOG.info("Start value={}", startValue); return startValue; }
@Override public byte[] prefixLookup(byte[] key, int prefixBytes) { if (prefixBytes != NodeKeyCompositor.PREFIX_BYTES) throw new RuntimeException("RocksDbDataSource.prefixLookup() supports only " + prefixBytes + "-bytes prefix"); resetDbLock.readLock().lock(); try { if (logger.isTraceEnabled()) logger.trace("~> RocksDbDataSource.prefixLookup(): " + name + ", key: " + toHexString(key)); // RocksDB sets initial position of iterator to the first key which is greater or equal to the seek key // since keys in RocksDB are ordered in asc order iterator must be initiated with the lowest key // thus bytes with indexes greater than PREFIX_BYTES must be nullified byte[] prefix = new byte[NodeKeyCompositor.PREFIX_BYTES]; arraycopy(key, 0, prefix, 0, NodeKeyCompositor.PREFIX_BYTES); byte[] ret = null; try (RocksIterator it = db.newIterator(readOpts)) { it.seek(prefix); if (it.isValid()) ret = it.value(); } catch (Exception e) { logger.error("Failed to seek by prefix in db '{}'", name, e); hintOnTooManyOpenFiles(e); throw new RuntimeException(e); } if (logger.isTraceEnabled()) logger.trace("<~ RocksDbDataSource.prefixLookup(): " + name + ", key: " + toHexString(key) + ", " + (ret == null ? "null" : ret.length)); return ret; } finally { resetDbLock.readLock().unlock(); } }
@Override public String[] next() { counter++; if (counter % 100000 == 0) { logger.info("scanned {} rows from rocksDB", counter); } String[] result = rowEncoder.decode(new KV(rocksIterator.key(), rocksIterator.value())); rocksIterator.next(); return result; }
@Override public Block next() { try { return new Block(Longs.fromByteArray(iter.key()), BlockMeta.parseFrom(iter.value())); } catch (Exception e) { throw new RuntimeException(e); } finally { iter.next(); } } };
@Override public List<BlockLocation> getLocations(long id) { RocksIterator iter = mDb.newIterator(mBlockLocationsColumn, new ReadOptions().setPrefixSameAsStart(true)); iter.seek(Longs.toByteArray(id)); List<BlockLocation> locations = new ArrayList<>(); for (; iter.isValid(); iter.next()) { try { locations.add(BlockLocation.parseFrom(iter.value())); } catch (Exception e) { throw new RuntimeException(e); } } return locations; }
@Override public Status scan(final String table, final String startkey, final int recordcount, final Set<String> fields, final Vector<HashMap<String, ByteIterator>> result) { try { if (!COLUMN_FAMILIES.containsKey(table)) { createColumnFamily(table); } final ColumnFamilyHandle cf = COLUMN_FAMILIES.get(table).getHandle(); try(final RocksIterator iterator = rocksDb.newIterator(cf)) { int iterations = 0; for (iterator.seek(startkey.getBytes(UTF_8)); iterator.isValid() && iterations < recordcount; iterator.next()) { final HashMap<String, ByteIterator> values = new HashMap<>(); deserializeValues(iterator.value(), fields, values); result.add(values); iterations++; } } return Status.OK; } catch(final RocksDBException e) { LOGGER.error(e.getMessage(), e); return Status.ERROR; } }
@Override public Set<EdgeEntry> allEdges() { Set<EdgeEntry> edges = new HashSet<>(); RocksIterator iter = mDb.newIterator(mEdgesColumn); iter.seekToFirst(); while (iter.isValid()) { long parentId = RocksUtils.readLong(iter.key(), 0); String childName = new String(iter.key(), Longs.BYTES, iter.key().length - Longs.BYTES); long childId = Longs.fromByteArray(iter.value()); edges.add(new EdgeEntry(parentId, childName, childId)); iter.next(); } return edges; }
MutableInode<?> inode; try { inode = MutableInode.fromProto(InodeMeta.Inode.parseFrom(inodeIter.value())); } catch (Exception e) { throw new RuntimeException(e); System.arraycopy(key, Longs.BYTES, name, 0, key.length - Longs.BYTES); sb.append(String.format("<%s,%s>->%s%n", Longs.fromByteArray(id), new String(name), Longs.fromByteArray(edgeIter.value()))); edgeIter.next();
@Override public <T> List<Pair<String, T>> fetchTop(int size, Type type) throws FailStoreException { RocksIterator iterator = null; try { List<Pair<String, T>> list = new ArrayList<Pair<String, T>>(size); iterator = db.newIterator(); for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { iterator.status(); String key = new String(iterator.key(), "UTF-8"); T value = JSON.parse(new String(iterator.value(), "UTF-8"), type); Pair<String, T> pair = new Pair<String, T>(key, value); list.add(pair); if (list.size() >= size) { break; } } return list; } catch (Exception e) { throw new FailStoreException(e); } finally { if (iterator != null) { iterator.dispose(); } } }
@Override public <T> List<Pair<String, T>> fetchTop(int size, Type type) throws FailStoreException { RocksIterator iterator = null; try { List<Pair<String, T>> list = new ArrayList<Pair<String, T>>(size); iterator = db.newIterator(); for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { iterator.status(); String key = new String(iterator.key(), "UTF-8"); T value = JSON.parse(new String(iterator.value(), "UTF-8"), type); Pair<String, T> pair = new Pair<String, T>(key, value); list.add(pair); if (list.size() >= size) { break; } } return list; } catch (Exception e) { throw new FailStoreException(e); } finally { if (iterator != null) { iterator.dispose(); } } }
protected boolean makeMore() { if(innerIterator.isValid()) { byte[] keyEntry = innerIterator.key(); byte[] valueEntry = innerIterator.value(); innerIterator.next(); if (RocksDbStorageEngine.this.isPartitionScanSupported()) { keyEntry = StoreBinaryFormat.extractKey(keyEntry); } ByteArray key = new ByteArray(keyEntry); for(Versioned<byte[]> val: StoreBinaryFormat.fromByteArray(valueEntry)) { cache.add(new Pair<ByteArray, Versioned<byte[]>>(key, val)); } return true; } return false; }
protected boolean makeMore() { if(innerIterator.isValid()) { byte[] keyEntry = innerIterator.key(); // Check if the next key returned starts with the expected // prefix if(StoreBinaryFormat.extractPartition(keyEntry) != partitionId) { return false; } byte[] valueEntry = innerIterator.value(); innerIterator.next(); ByteArray key = new ByteArray(StoreBinaryFormat.extractKey(keyEntry)); for(Versioned<byte[]> val: StoreBinaryFormat.fromByteArray(valueEntry)) { cache.add(new Pair<ByteArray, Versioned<byte[]>>(key, val)); } return true; } return false; }
void scanRange(RocksDbKey start, RocksDbKey end, RocksDbScanCallback fn) { try (ReadOptions ro = new ReadOptions()) { ro.setTotalOrderSeek(true); RocksIterator iterator = db.newIterator(ro); for (iterator.seek(start.getRaw()); iterator.isValid(); iterator.next()) { RocksDbKey key = new RocksDbKey(iterator.key()); if (key.compareTo(end) >= 0) { // past limit, quit return; } RocksDbValue val = new RocksDbValue(iterator.value()); if (!fn.cb(key, val)) { // if cb returns false, we are done with this section of rows return; } } } }
RocksDbValue val = new RocksDbValue(iterator.value());
@Override public BackendColumn next() { if (!this.matched) { if (!this.hasNext()) { throw new NoSuchElementException(); } } BackendColumn col = BackendColumn.of(this.itor.key(), this.itor.value()); this.itor.next(); this.matched = false; return col; }
/** * Just for debug */ @SuppressWarnings("unused") private void dump() { this.seek(); System.out.println(">>>> scan from " + this.table + ": " + (this.keyBegin == null ? "*" : StringEncoding.format(this.keyBegin)) + (this.itor.isValid() ? "" : " - No data")); for (; this.itor.isValid(); this.itor.next()) { System.out.println(String.format("%s=%s", StringEncoding.format(this.itor.key()), StringEncoding.format(this.itor.value()))); } }
@Override public int identifier() { byte[] value = rocksIterator.value(); return ByteBuffer.wrap(value).getInt(); }
@Override public T next() { if (closed) { throw new NoSuchElementException(); } T retVal = readRow(it.key(), it.value()); it.next(); return retVal; }
@Benchmark public void readRev(final Reader r, final Blackhole bh) { final RocksIterator iterator = r.db.newIterator(); iterator.seekToLast(); while (iterator.isValid()) { bh.consume(iterator.value()); iterator.prev(); } }