"Either the write lock must be held or the engine must be currently be failing itself"; try { this.versionMap.clear(); if (internalSearcherManager != null) { internalSearcherManager.removeListener(versionMap);
final Collection<DeleteVersionValue> getDeletedTombstones() { return versionMap.getAllTombstones().values(); }
/** * Returns the live version (add or delete) for this uid. */ VersionValue getUnderLock(final BytesRef uid) { return getUnderLock(uid, maps); }
private VersionValue getVersionFromMap(BytesRef id) { if (versionMap.isUnsafe()) { synchronized (versionMap) { // we are switching from an unsafe map to a safe map. This might happen concurrently // but we only need to do this once since the last operation per ID is to add to the version // map so once we pass this point we can safely lookup from the version map. if (versionMap.isUnsafe()) { refresh("unsafe_version_map", SearcherScope.INTERNAL); } versionMap.enforceSafeAccess(); } } return versionMap.getUnderLock(id); }
void putIndexUnderLock(BytesRef uid, IndexVersionValue version) { assert assertKeyedLockHeldByCurrentThread(uid); assert uid.bytes.length == uid.length : "Oversized _uid! UID length: " + uid.length + ", bytes length: " + uid.bytes.length; maps.put(uid, version); removeTombstoneUnderLock(uid); }
void putDeleteUnderLock(BytesRef uid, DeleteVersionValue version) { assert assertKeyedLockHeldByCurrentThread(uid); assert uid.bytes.length == uid.length : "Oversized _uid! UID length: " + uid.length + ", bytes length: " + uid.bytes.length; putTombstone(uid, version); maps.remove(uid, version); }
/** * Adds this uid/version to the pending adds map iff the map needs safe access. */ void maybePutIndexUnderLock(BytesRef uid, IndexVersionValue version) { assert assertKeyedLockHeldByCurrentThread(uid); Maps maps = this.maps; if (maps.isSafeAccessMode()) { putIndexUnderLock(uid, version); } else { // Even though we don't store a record of the indexing operation (and mark as unsafe), // we should still remove any previous delete for this uuid (avoid accidental accesses). // Not this should not hurt performance because the tombstone is small (or empty) when unsafe is relevant. removeTombstoneUnderLock(uid); maps.current.markAsUnsafe(); assert putAssertionMap(uid, version); } }
private void pruneDeletedTombstones() { long timeMSec = engineConfig.getThreadPool().relativeTimeInMillis(); // TODO: not good that we reach into LiveVersionMap here; can we move this inside VersionMap instead? problem is the dirtyLock... // we only need to prune the deletes map; the current/old version maps are cleared on refresh: for (Map.Entry<BytesRef, VersionValue> entry : versionMap.getAllTombstones()) { BytesRef uid = entry.getKey(); try (Releasable ignored = acquireLock(uid)) { // can we do it without this lock on each value? maybe batch to a set and get the lock once per set? // Must re-get it here, vs using entry.getValue(), in case the uid was indexed/deleted since we pulled the iterator: VersionValue versionValue = versionMap.getTombstoneUnderLock(uid); if (versionValue != null) { if (timeMSec - versionValue.getTime() > getGcDeletesInMillis()) { versionMap.removeTombstoneUnderLock(uid); } } } } lastDeleteVersionPruneTimeMSec = timeMSec; }
this.versionMap = new LiveVersionMap(); store.incRef(); IndexWriter writer = null; manager = createSearcherManager(); this.searcherManager = manager; this.versionMap.setManager(searcherManager); assert pendingTranslogRecovery.get() == false : "translog recovery can't be pending before we set it"; if (success == false) { IOUtils.closeWhileHandlingException(writer, translog, manager, scheduler); versionMap.clear(); if (isClosed.get() == false) {
@Override public long getIndexBufferRAMBytesUsed() { // We don't guard w/ readLock here, so we could throw AlreadyClosedException return indexWriter.ramBytesUsed() + versionMap.ramBytesUsedForRefresh(); }
/** * Try to prune tombstones whose timestamp is less than maxTimestampToPrune and seqno at most the maxSeqNoToPrune. */ void pruneTombstones(long maxTimestampToPrune, long maxSeqNoToPrune) { for (Map.Entry<BytesRef, DeleteVersionValue> entry : tombstones.entrySet()) { // we do check before we actually lock the key - this way we don't need to acquire the lock for tombstones that are not // prune-able. If the tombstone changes concurrently we will re-read and step out below since if we can't collect it now w // we won't collect the tombstone below since it must be newer than this one. if (canRemoveTombstone(maxTimestampToPrune, maxSeqNoToPrune, entry.getValue())) { final BytesRef uid = entry.getKey(); try (Releasable lock = keyedLock.tryAcquire(uid)) { // we use tryAcquire here since this is a best effort and we try to be least disruptive // this method is also called under lock in the engine under certain situations such that this can lead to deadlocks // if we do use a blocking acquire. see #28714 if (lock != null) { // did we get the lock? // Must re-get it here, vs using entry.getValue(), in case the uid was indexed/deleted since we pulled the iterator: final DeleteVersionValue versionValue = tombstones.get(uid); if (versionValue != null) { if (canRemoveTombstone(maxTimestampToPrune, maxSeqNoToPrune, versionValue)) { removeTombstoneUnderLock(uid); } } } } } } }
private boolean innerIndex(Index index) throws IOException { try (Releasable ignored = acquireLock(index.uid())) { final long currentVersion; VersionValue versionValue = versionMap.getUnderLock(index.uid().bytes()); if (versionValue == null) { currentVersion = loadCurrentVersionFromIndex(index.uid()); versionMap.putUnderLock(index.uid().bytes(), new VersionValue(updatedVersion, translogLocation)); index.setTranslogLocation(translogLocation); indexingService.postIndexUnderLock(index);
@Override protected final void writerSegmentStats(SegmentsStats stats) { stats.addVersionMapMemoryInBytes(versionMap.ramBytesUsed()); stats.addIndexWriterMemoryInBytes(indexWriter.ramBytesUsed()); stats.updateMaxUnsafeAutoIdTimestamp(maxUnsafeAutoIdTimestamp.get()); }
private boolean putAssertionMap(BytesRef uid, IndexVersionValue version) { assert assertKeyedLockHeldByCurrentThread(uid); assert uid.bytes.length == uid.length : "Oversized _uid! UID length: " + uid.length + ", bytes length: " + uid.bytes.length; unsafeKeysMap.put(uid, version); return true; }
@Override public DeleteResult delete(Delete delete) throws IOException { versionMap.enforceSafeAccess(); assert Objects.equals(delete.uid().field(), uidField) : delete.uid().field(); assert assertVersionType(delete); final DeleteResult deleteResult; try (ReleasableLock ignored = readLock.acquire(); Releasable ignored2 = versionMap.acquireLock(delete.uid().bytes())) { ensureOpen(); lastWriteNanos = delete.startTime();
assert assertIncomingSequenceNumber(index.origin(), index.seqNo()); assert assertVersionType(index); try (Releasable ignored = versionMap.acquireLock(index.uid().bytes()); Releasable indexThrottle = doThrottle ? () -> {} : throttle.acquireThrottle()) { lastWriteNanos = index.startTime(); versionMap.maybePutIndexUnderLock(index.uid().bytes(), new IndexVersionValue(translogLocation, plan.versionForIndexing, plan.seqNoForIndexing, index.primaryTerm()));
private final LiveVersionMap versionMap = new LiveVersionMap();
if (get.realtime()) { VersionValue versionValue = null; try (Releasable ignore = versionMap.acquireLock(get.uid().bytes())) {
/** * Adds this uid/version to the pending adds map iff the map needs safe access. */ void maybePutIndexUnderLock(BytesRef uid, IndexVersionValue version) { assert assertKeyedLockHeldByCurrentThread(uid); Maps maps = this.maps; if (maps.isSafeAccessMode()) { putIndexUnderLock(uid, version); } else { // Even though we don't store a record of the indexing operation (and mark as unsafe), // we should still remove any previous delete for this uuid (avoid accidental accesses). // Not this should not hurt performance because the tombstone is small (or empty) when unsafe is relevant. removeTombstoneUnderLock(uid); maps.current.markAsUnsafe(); assert putAssertionMap(uid, version); } }
private void pruneDeletedTombstones() { long timeMSec = engineConfig.getThreadPool().estimatedTimeInMillis(); // TODO: not good that we reach into LiveVersionMap here; can we move this inside VersionMap instead? problem is the dirtyLock... // we only need to prune the deletes map; the current/old version maps are cleared on refresh: for (Map.Entry<BytesRef, VersionValue> entry : versionMap.getAllTombstones()) { BytesRef uid = entry.getKey(); try (Releasable ignored = acquireLock(uid)) { // can we do it without this lock on each value? maybe batch to a set and get the lock once // per set? // Must re-get it here, vs using entry.getValue(), in case the uid was indexed/deleted since we pulled the iterator: VersionValue versionValue = versionMap.getTombstoneUnderLock(uid); if (versionValue != null) { if (timeMSec - versionValue.time() > engineConfig.getGcDeletesInMillis()) { versionMap.removeTombstoneUnderLock(uid); } } } } lastDeleteVersionPruneTimeMSec = timeMSec; }