protected Cursor openCursor(Transaction txn) { return urlsDB.openCursor(txn, null); }
@Override public ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entries() { try { Cursor cursor = getBdbDatabase().openCursor(null, null); // evict data brought in by the cursor walk right away if(this.minimizeScanImpact) cursor.setCacheMode(CacheMode.EVICT_BIN); return new BdbEntriesIterator(cursor, this); } catch(DatabaseException e) { this.bdbEnvironmentStats.reportException(e); logger.error(e); throw new PersistenceFailureException(e); } }
@Override public ClosableIterator<ByteArray> keys(int partition) { try { Cursor cursor = getBdbDatabase().openCursor(null, null); // evict data brought in by the cursor walk right away if(this.minimizeScanImpact) cursor.setCacheMode(CacheMode.EVICT_BIN); return new BdbPartitionKeysIterator(cursor, partition, this); } catch(DatabaseException e) { this.bdbEnvironmentStats.reportException(e); logger.error(e); throw new PersistenceFailureException(e); } }
@Override public ClosableIterator<ByteArray> keys() { try { Cursor cursor = getBdbDatabase().openCursor(null, null); // evict data brought in by the cursor walk right away if(this.minimizeScanImpact) cursor.setCacheMode(CacheMode.EVICT_BIN); return new BdbKeysIterator(cursor, this); } catch(DatabaseException e) { this.bdbEnvironmentStats.reportException(e); logger.error(e); throw new PersistenceFailureException(e); } }
@Override public ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> entries(int partition) { try { Cursor cursor = getBdbDatabase().openCursor(null, null); // evict data brought in by the cursor walk right away if(this.minimizeScanImpact) cursor.setCacheMode(CacheMode.EVICT_BIN); return new BdbPartitionEntriesIterator(cursor, partition, this); } catch(DatabaseException e) { this.bdbEnvironmentStats.reportException(e); logger.error(e); throw new PersistenceFailureException(e); } }
@Override public void transfer() throws Exception { cursor = srcDB.openCursor(null, null); DatabaseEntry keyEntry = new DatabaseEntry(); DatabaseEntry valueEntry = new DatabaseEntry(); List<Versioned<byte[]>> vals; long startTime = System.currentTimeMillis(); int scanCount = 0; int keyCount = 0; while(cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) { keyCount++; vals = StoreBinaryFormat.fromByteArray(valueEntry.getData()); scanCount += vals.size(); // pull out the real key byte[] stripedKey = StoreBinaryFormat.extractKey(keyEntry.getData()); OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(stripedKey), valueEntry); if(OperationStatus.SUCCESS != putStatus) { String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(keyEntry.getData()); logger.error(errorStr); throw new Exception(errorStr); } if(scanCount % 1000000 == 0) logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs"); } logger.info("Reverted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs"); }
/** * @return the key to the first item in the database * @throws DatabaseException */ protected DatabaseEntry getFirstKey() throws DatabaseException { DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); Cursor cursor = pendingUrisDB.openCursor(null,null); OperationStatus status = cursor.getNext(key,value,null); cursor.close(); if(status == OperationStatus.SUCCESS) { return key; } return null; }
protected OperationStatus getNextNearestItem(DatabaseEntry headKey, DatabaseEntry result) throws DatabaseException { Cursor cursor = null; OperationStatus status; try { cursor = this.pendingUrisDB.openCursor(null, null); // get cap; headKey at this point should always point to // a queue-beginning cap entry (zero-length value) status = cursor.getSearchKey(headKey, result, null); if (status != OperationStatus.SUCCESS) { LOGGER.severe("bdb queue cap missing: " + status.toString() + " " + new String(headKey.getData())); return status; } if (result.getData().length > 0) { LOGGER.severe("bdb queue has nonzero size: " + result.getData().length); return OperationStatus.KEYEXIST; } // get next item (real first item of queue) status = cursor.getNext(headKey,result,null); } finally { if(cursor!=null) { cursor.close(); } } return status; }
List<Pair<String, T>> list = new ArrayList<Pair<String, T>>(); cursor = db.openCursor(null, CursorConfig.DEFAULT); DatabaseEntry foundKey = new DatabaseEntry(); DatabaseEntry foundValue = new DatabaseEntry();
List<Pair<String, T>> list = new ArrayList<Pair<String, T>>(); cursor = db.openCursor(null, CursorConfig.DEFAULT); DatabaseEntry foundKey = new DatabaseEntry(); DatabaseEntry foundValue = new DatabaseEntry();
@Override public void transfer() throws Exception { cursor = srcDB.openCursor(null, null); DatabaseEntry keyEntry = new DatabaseEntry(); DatabaseEntry valueEntry = new DatabaseEntry();
@Override public void transfer() throws Exception { cursor = srcDB.openCursor(null, null); DatabaseEntry keyEntry = new DatabaseEntry(); DatabaseEntry valueEntry = new DatabaseEntry();
@Override public void transfer() throws Exception { cursor = srcDB.openCursor(null, null); DatabaseEntry keyEntry = new DatabaseEntry(); DatabaseEntry valueEntry = new DatabaseEntry();
Cursor cursor = null; try { cursor = pendingUrisDB.openCursor(null, null); OperationStatus result = cursor.getSearchKeyRange(headKey, value, null);
@Override public void transfer() throws Exception { cursor = srcDB.openCursor(null, null); DatabaseEntry keyEntry = new DatabaseEntry(); DatabaseEntry valueEntry = new DatabaseEntry();
DatabaseEntry foundData = new DatabaseEntry(); cursor = db.openCursor(tx, null); OperationStatus status = cursor.getSearchKeyRange(foundKey, foundData, getLockMode(txh));
@Override public void transfer() throws Exception { cursor = srcDB.openCursor(null, null); DatabaseEntry keyEntry = new DatabaseEntry(); DatabaseEntry valueEntry = new DatabaseEntry();
/** * Utility method to perform action for all pending CrawlURI instances. * @param c Closure action to perform * @throws DatabaseException */ protected void forAllPendingDo(Closure c) throws DatabaseException { DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); Cursor cursor = pendingUrisDB.openCursor(null, null); while (cursor.getNext(key, value, null) == OperationStatus.SUCCESS) { if (value.getData().length == 0) { continue; } CrawlURI item = (CrawlURI) crawlUriBinding.entryToObject(value); c.execute(item); } cursor.close(); } }
byte[] schemeAuthorityKeyBytes = key.getData(); Cursor cursor = alreadySeen.openCursor(null, null); long forgottenCount = 0l;
DatabaseEntry value = new DatabaseEntry(); Transaction tnx = env.beginTransaction(null, null); Cursor cursor = statisticsDB.openCursor(tnx, null); result = cursor.getFirst(key, value, null);