/** * @return the key to the first item in the database * @throws DatabaseException */ protected DatabaseEntry getFirstKey() throws DatabaseException { DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); Cursor cursor = pendingUrisDB.openCursor(null,null); OperationStatus status = cursor.getNext(key,value,null); cursor.close(); if(status == OperationStatus.SUCCESS) { return key; } return null; }
@Override public void transfer() throws Exception { cursor = srcDB.openCursor(null, null); DatabaseEntry keyEntry = new DatabaseEntry(); DatabaseEntry valueEntry = new DatabaseEntry(); List<Versioned<byte[]>> vals; long startTime = System.currentTimeMillis(); int scanCount = 0; int keyCount = 0; while(cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED) == OperationStatus.SUCCESS) { keyCount++; vals = StoreBinaryFormat.fromByteArray(valueEntry.getData()); scanCount += vals.size(); // pull out the real key byte[] stripedKey = StoreBinaryFormat.extractKey(keyEntry.getData()); OperationStatus putStatus = dstDB.put(null, new DatabaseEntry(stripedKey), valueEntry); if(OperationStatus.SUCCESS != putStatus) { String errorStr = "Put failed with " + putStatus + " for key" + BdbConvertData.writeAsciiString(keyEntry.getData()); logger.error(errorStr); throw new Exception(errorStr); } if(scanCount % 1000000 == 0) logger.info("Reverted " + scanCount + " entries in " + (System.currentTimeMillis() - startTime) / 1000 + " secs"); } logger.info("Reverted " + scanCount + " entries and " + keyCount + " keys in " + (System.currentTimeMillis() - startTime) / 1000 + " secs"); }
DatabaseEntry keyEntry = new DatabaseEntry(); DatabaseEntry valueEntry = new DatabaseEntry(); OperationStatus status; valueEntry.setPartial(true); try { if(!positioned) { positioned = true; keyEntry.setData(StoreBinaryFormat.makePartitionKey(partition)); status = cursor.getSearchKeyRange(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED); } else { status = cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED); if(StoreBinaryFormat.extractPartition(keyEntry.getData()) != partition) return false; current = new ByteArray(StoreBinaryFormat.extractKey(keyEntry.getData())); return true; } catch(DatabaseException e) {
@Override public void delete(String key) throws FailStoreException { try { DatabaseEntry delKey = new DatabaseEntry(); delKey.setData(key.getBytes("UTF-8")); @SuppressWarnings("unused") OperationStatus status = db.delete(null, delKey); } catch (Exception e) { throw new FailStoreException(e); } }
/** * Returns the docid of an already seen url. * * @param url the URL for which the docid is returned. * @return the docid of the url if it is seen before. Otherwise -1 is returned. */ public int getDocId(String url) { synchronized (mutex) { OperationStatus result = null; DatabaseEntry value = new DatabaseEntry(); try { DatabaseEntry key = new DatabaseEntry(url.getBytes()); result = docIDsDB.get(null, key, value, null); } catch (RuntimeException e) { if (config.isHaltOnError()) { throw e; } else { logger.error("Exception thrown while getting DocID", e); return -1; } } if ((result == OperationStatus.SUCCESS) && (value.getData().length > 0)) { return Util.byteArray2Int(value.getData()); } return -1; } }
long schemeAuthorityKeyLong = calcSchemeAuthorityKeyBytes(url); DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); byte[] schemeAuthorityKeyBytes = key.getData(); Cursor cursor = alreadySeen.openCursor(null, null); long forgottenCount = 0l; for (OperationStatus status = cursor.getSearchKeyRange(key, value, null); status == OperationStatus.SUCCESS; status = cursor.getNext(key, value, null)) { byte[] keyData = key.getData(); if (keyData[0] == schemeAuthorityKeyBytes[0] && keyData[1] == schemeAuthorityKeyBytes[1] && keyData[2] == schemeAuthorityKeyBytes[2]) { cursor.delete(); forgottenCount++; } else {
try { transaction = this.environment.beginTransaction(null, null); DatabaseEntry keyEntry = new DatabaseEntry(key.get()); OperationStatus status = getBdbDatabase().delete(transaction, keyEntry); if(OperationStatus.SUCCESS == status) return true; } else { DatabaseEntry valueEntry = new DatabaseEntry(); OperationStatus status = getBdbDatabase().get(transaction, keyEntry, valueEntry, return false; List<Versioned<byte[]>> vals = StoreBinaryFormat.fromByteArray(valueEntry.getData()); Iterator<Versioned<byte[]>> iter = vals.iterator(); int numVersions = vals.size(); valueEntry.setData(StoreBinaryFormat.toByteArray(vals)); getBdbDatabase().put(transaction, keyEntry, valueEntry); } else {
@Override public byte[] lastKey() { Cursor c = bdb.openCursor(null, CursorConfig.READ_UNCOMMITTED); try { DatabaseEntry dk = new DatabaseEntry(); if (c.getLast(dk, new DatabaseEntry(), lockMode) == opSuccess) { return dk.getData(); } return null; } finally { c.close(); } }
/** * internal/raw bdb delete */ @Override public byte[] delete(byte[] key) throws DatabaseException { DatabaseEntry dk = new DatabaseEntry(key); DatabaseEntry dv = new DatabaseEntry(); dv.setPartial(0, 0, true); Cursor cursor = bdb.openCursor(null, CursorConfig.READ_UNCOMMITTED); try { if (cursor.getSearchKey(dk, dv, lockMode) == opSuccess && cursor.delete() == opSuccess && cursor.getPrev(dk, dv, lockMode) == opSuccess) { return dk.getData(); } } finally { cursor.close(); } return null; }
@Override protected boolean db_exists(JE_Transaction jetxn, byte[] key, boolean rmw) throws Exception { DatabaseEntry keyEntry = new DatabaseEntry(key); DatabaseEntry dataEntry = new DatabaseEntry(); dataEntry.setPartial(0, 0, true); OperationStatus status = mDatabase.get (jetxn == null ? null : jetxn.mTxn, keyEntry, dataEntry, rmw ? LockMode.RMW : null); return status != OperationStatus.NOTFOUND; }
/** * Add a dummy 'cap' entry at the given insertion key. Prevents * 'seeks' to queue heads from holding lock on last item of * 'preceding' queue. See: * http://sourceforge.net/tracker/index.php?func=detail&aid=1262665&group_id=73833&atid=539102 * * @param origin key at which to insert the cap */ public void addCap(byte[] origin) { try { pendingUrisDB.put(null, new DatabaseEntry(origin), new DatabaseEntry(new byte[0])); } catch (DatabaseException e) { throw new RuntimeException(e); } }
private boolean fetchNextKey() { DatabaseEntry keyEntry = new DatabaseEntry(); DatabaseEntry valueEntry = new DatabaseEntry(); valueEntry.setPartial(true); try { OperationStatus status = cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED); if(OperationStatus.NOTFOUND == status) { // we have reached the end of the cursor return false; } if(bdbEngine.isPartitionScanSupported()) current = new ByteArray(StoreBinaryFormat.extractKey(keyEntry.getData())); else current = new ByteArray(keyEntry.getData()); return true; } catch(DatabaseException e) { bdbEngine.bdbEnvironmentStats.reportException(e); logger.error(e); throw new PersistenceFailureException(e); } } }
public List<WebURL> get(int max) { synchronized (mutex) { List<WebURL> results = new ArrayList<>(max); DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); Transaction txn = beginTransaction(); try (Cursor cursor = openCursor(txn)) { OperationStatus result = cursor.getFirst(key, value, null); int matches = 0; while ((matches < max) && (result == OperationStatus.SUCCESS)) { if (value.getData().length > 0) { results.add(webURLBinding.entryToObject(value)); matches++; } result = cursor.getNext(key, value, null); } } commit(txn); return results; } }
@Override public List<byte[]> getDuplicates(String tableName, byte[] keyBytes, DatabaseSession databaseSession) throws BimserverDatabaseException { DatabaseEntry key = new DatabaseEntry(keyBytes); DatabaseEntry value = new DatabaseEntry(); try { TableWrapper tableWrapper = getTableWrapper(tableName); Cursor cursor = tableWrapper.getDatabase().openCursor(getTransaction(databaseSession, tableWrapper), getCursorConfig(tableWrapper)); try { OperationStatus operationStatus = cursor.getSearchKey(key, value, LockMode.DEFAULT); List<byte[]> result = new ArrayList<byte[]>(); while (operationStatus == OperationStatus.SUCCESS) { result.add(value.getData()); operationStatus = cursor.getNextDup(key, value, LockMode.DEFAULT); } return result; } finally { cursor.close(); } } catch (DatabaseException e) { LOGGER.error("", e); } return null; }
protected boolean makeMore() { DatabaseEntry keyEntry = new DatabaseEntry(); DatabaseEntry valueEntry = new DatabaseEntry(); try { OperationStatus status = cursor.getNext(keyEntry, valueEntry, LockMode.READ_UNCOMMITTED); if(OperationStatus.NOTFOUND == status) { // we have reached the end of the cursor return false; } ByteArray key = null; if(bdbEngine.isPartitionScanSupported()) key = new ByteArray(StoreBinaryFormat.extractKey(keyEntry.getData())); else key = new ByteArray(keyEntry.getData()); for(Versioned<byte[]> val: StoreBinaryFormat.fromByteArray(valueEntry.getData())) this.cache.add(Pair.create(key, val)); return true; } catch(DatabaseException e) { bdbEngine.bdbEnvironmentStats.reportException(e); logger.error(e); throw new PersistenceFailureException(e); } } }
public void delete(int count) { synchronized (mutex) { DatabaseEntry key = new DatabaseEntry(); DatabaseEntry value = new DatabaseEntry(); Transaction txn = beginTransaction(); try (Cursor cursor = openCursor(txn)) { OperationStatus result = cursor.getFirst(key, value, null); int matches = 0; while ((matches < count) && (result == OperationStatus.SUCCESS)) { cursor.delete(); matches++; result = cursor.getNext(key, value, null); } } commit(txn); } }
@Override public Storable findEntry(StorableKey key) throws StoreException { DatabaseEntry result = new DatabaseEntry(); try { OperationStatus status = _entries.get(null, dbKey(key), result, null); if (status != OperationStatus.SUCCESS) { return null; } return _storableConverter.decode(key, result.getData(), result.getOffset(), result.getSize()); } catch (DatabaseException de) { return _convertDBE(key, de); } }
@Override public void delete(String indexTableName, byte[] featureBytesOldIndex, byte[] array, DatabaseSession databaseSession) throws BimserverLockConflictException { try { TableWrapper tableWrapper = getTableWrapper(indexTableName); Cursor cursor = tableWrapper.getDatabase().openCursor(getTransaction(databaseSession, tableWrapper), getCursorConfig(tableWrapper)); try { if (cursor.getSearchBoth(new DatabaseEntry(featureBytesOldIndex), new DatabaseEntry(array), LockMode.DEFAULT) == OperationStatus.SUCCESS) { cursor.delete(); } } finally { cursor.close(); } } catch (LockConflictException e) { throw new BimserverLockConflictException(e); } catch (DatabaseException e) { LOGGER.error("", e); } catch (UnsupportedOperationException e) { LOGGER.error("", e); } catch (IllegalArgumentException e) { LOGGER.error("", e); } catch (BimserverDatabaseException e) { LOGGER.error("", e); } }
final BdbMultipleWorkQueues queues = ((BdbFrontier) frontier) .getWorkQueues(); DatabaseEntry key = new DatabaseEntry(origin); CrawlURI curi = null; int tries = 1; if(!ArchiveUtils.startsWith(key.getData(),origin)) { LOGGER.severe( "inconsistency: "+classKey+"("+ getPrefixClassKey(origin)+") with " + getCount() + " items gave " + curi +"("+getPrefixClassKey(key.getData())); key.setData(origin); + " in queue " + classKey + " with " + getCount() + " items using key " + getPrefixClassKey(key.getData()));