@Override public void transfer() throws Exception { cursor = srcDB.openCursor(null, null); DatabaseEntry keyEntry = new DatabaseEntry(); DatabaseEntry valueEntry = new DatabaseEntry(); List<Versioned<byte[]>> vals; long startTime = System.currentTimeMillis(); int scanCount = 0; int keyCount = 0; while(cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) { keyCount++; vals = StoreBinaryFormat.fromByteArray(valueEntry.getData()); scanCount += vals.size(); // pull out the real key byte[] stripedKey = StoreBinaryFormat.extractKey(keyEntry.getData()); OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(stripedKey), valueEntry); if(OperationStatus.SUCCESS != putStatus) { String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(keyEntry.getData()); logger.error(errorStr); throw new Exception(errorStr); } if(scanCount % 1000000 == 0) logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs"); } logger.info("Reverted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs"); }
/** * @return the key to the first item in the database * @throws DatabaseException */ protected DatabaseEntry getFirstKey() throws DatabaseException { DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); Cursor cursor = pendingUrisDB.openCursor(null,null); OperationStatus status = cursor.getNext(key,value,null); cursor.close(); if(status == OperationStatus.SUCCESS) { return key; } return null; }
public List<WebURL> get(int max) { synchronized (mutex) { List<WebURL> results = new ArrayList<>(max); DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); Transaction txn = beginTransaction(); try (Cursor cursor = openCursor(txn)) { OperationStatus result = cursor.getFirst(key, value, null); int matches = 0; while ((matches < max) && (result == OperationStatus.SUCCESS)) { if (value.getData().length > 0) { results.add(webURLBinding.entryToObject(value)); matches++; } result = cursor.getNext(key, value, null); } } commit(txn); return results; } }
protected OperationStatus getNextNearestItem(DatabaseEntry headKey, DatabaseEntry result) throws DatabaseException { Cursor cursor = null; OperationStatus status; try { cursor = this.pendingUrisDB.openCursor(null, null); // get cap; headKey at this point should always point to // a queue-beginning cap entry (zero-length value) status = cursor.getSearchKey(headKey, result, null); if (status != OperationStatus.SUCCESS) { LOGGER.severe("bdb queue cap missing: " + status.toString() + " " + new String(headKey.getData())); return status; } if (result.getData().length > 0) { LOGGER.severe("bdb queue has nonzero size: " + result.getData().length); return OperationStatus.KEYEXIST; } // get next item (real first item of queue) status = cursor.getNext(headKey,result,null); } finally { if(cursor!=null) { cursor.close(); } } return status; }
DatabaseEntry foundKey = new DatabaseEntry(); DatabaseEntry foundValue = new DatabaseEntry(); while (cursor.getNext(foundKey, foundValue, LockMode.DEFAULT) == OperationStatus.SUCCESS) { String key = new String(foundKey.getData(), "UTF-8");
DatabaseEntry foundKey = new DatabaseEntry(); DatabaseEntry foundValue = new DatabaseEntry(); while (cursor.getNext(foundKey, foundValue, LockMode.DEFAULT) == OperationStatus.SUCCESS) { String key = new String(foundKey.getData(), "UTF-8");
while(cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) { keyCount++;
int scanCount = 0; int keyCount = 0; while(cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) { keyCount++;
while(cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) { keyCount++;
result = cursor.getNext(key, value, null);
int scanCount = 0; int keyCount = 0; while(cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) { scanCount++; if(scanCount % 1000000 == 0)
public void delete(int count) { synchronized (mutex) { DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); Transaction txn = beginTransaction(); try (Cursor cursor = openCursor(txn)) { OperationStatus result = cursor.getFirst(key, value, null); int matches = 0; while ((matches < count) && (result == OperationStatus.SUCCESS)) { cursor.delete(); matches++; result = cursor.getNext(key, value, null); } } commit(txn); } }
break; status = cursor.getNext(foundKey, foundData, getLockMode(txh));
/** * Utility method to perform action for all pending CrawlURI instances. * @param c Closure action to perform * @throws DatabaseException */ protected void forAllPendingDo(Closure c) throws DatabaseException { DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); Cursor cursor = pendingUrisDB.openCursor(null, null); while (cursor.getNext(key, value, null) == OperationStatus.SUCCESS) { if (value.getData().length == 0) { continue; } CrawlURI item = (CrawlURI) crawlUriBinding.entryToObject(value); c.execute(item); } cursor.close(); } }
protected boolean makeMore() { DatabaseEntry keyEntry = new DatabaseEntry(); DatabaseEntry valueEntry = new DatabaseEntry(); try { OperationStatus status = cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED); if(OperationStatus.NOTFOUND == status) { // we have reached the end of the cursor return false; } ByteArray key = null; if(bdbEngine.isPartitionScanSupported()) key = new ByteArray(StoreBinaryFormat.extractKey(keyEntry.getData())); else key = new ByteArray(keyEntry.getData()); for(Versioned<byte[]> val: StoreBinaryFormat.fromByteArray(valueEntry.getData())) this.cache.add(Pair.create(key, val)); return true; } catch(DatabaseException e) { bdbEngine.bdbEnvironmentStats.reportException(e); logger.error(e); throw new PersistenceFailureException(e); } } }
status = cursor.getNext(key, value, null)) {
private boolean fetchNextKey() { DatabaseEntry keyEntry = new DatabaseEntry(); DatabaseEntry valueEntry = new DatabaseEntry(); valueEntry.setPartial(true); try { OperationStatus status = cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED); if(OperationStatus.NOTFOUND == status) { // we have reached the end of the cursor return false; } if(bdbEngine.isPartitionScanSupported()) current = new ByteArray(StoreBinaryFormat.extractKey(keyEntry.getData())); else current = new ByteArray(keyEntry.getData()); return true; } catch(DatabaseException e) { bdbEngine.bdbEnvironmentStats.reportException(e); logger.error(e); throw new PersistenceFailureException(e); } } }
counterValues.put(name, counterValue); result = cursor.getNext(key, value, null);
LockMode.READ_UNCOMMITTED); } else { status = cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED);
LockMode.READ_UNCOMMITTED); } else { status = cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED);