private void ensureWriteAllowed(Engine.Operation op) throws IllegalIndexShardStateException { Engine.Operation.Origin origin = op.origin(); IndexShardState state = this.state; // one time volatile read if (origin == Engine.Operation.Origin.PRIMARY) { if (writeAllowedStatesForPrimary.contains(state) == false) { throw new IllegalIndexShardStateException(shardId, state, "operation only allowed when shard state is one of " + writeAllowedStatesForPrimary + ", origin [" + origin + "]"); } } else if (origin.isRecovery()) { if (state != IndexShardState.RECOVERING) { throw new IllegalIndexShardStateException(shardId, state, "operation only allowed when recovering, origin [" + origin + "]"); } } else { assert origin == Engine.Operation.Origin.REPLICA; if (writeAllowedStatesForReplica.contains(state) == false) { throw new IllegalIndexShardStateException(shardId, state, "operation only allowed when shard state is one of " + writeAllowedStatesForReplica + ", origin [" + origin + "]"); } } }
private void applyOperation(Engine engine, Engine.Operation operation) throws IOException { switch (operation.operationType()) { case INDEX: Engine.Index engineIndex = (Engine.Index) operation; Mapping update = engineIndex.parsedDoc().dynamicMappingsUpdate(); if (engineIndex.parsedDoc().dynamicMappingsUpdate() != null) { recoveredTypes.compute(engineIndex.type(), (k, mapping) -> mapping == null ? update : mapping.merge(update, false)); } engine.index(engineIndex); break; case DELETE: engine.delete((Engine.Delete) operation); break; case NO_OP: engine.noOp((Engine.NoOp) operation); break; default: throw new IllegalStateException("No operation defined for [" + operation + "]"); } }
@Override public IndexResult index(Index index) throws IOException { assert Objects.equals(index.uid().field(), uidField) : index.uid().field(); final boolean doThrottle = index.origin().isRecovery() == false; try (ReleasableLock releasableLock = readLock.acquire()) { ensureOpen(); assert assertIncomingSequenceNumber(index.origin(), index.seqNo()); assert assertVersionType(index); try (Releasable ignored = versionMap.acquireLock(index.uid().bytes()); if (index.origin() == Operation.Origin.PRIMARY) { plan = planIndexingAsPrimary(index); } else {
private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnSeqNo(final Operation op) throws IOException { assert op.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "resolving ops based on seq# but no seqNo is found"; final OpVsLuceneDocStatus status; VersionValue versionValue = getVersionFromMap(op.uid().bytes()); assert incrementVersionLookup(); if (versionValue != null) { if (op.seqNo() > versionValue.seqNo) { status = OpVsLuceneDocStatus.OP_NEWER; } else if (op.seqNo() == versionValue.seqNo) { assert versionValue.term == op.primaryTerm() : "primary term not matched; id=" + op.id() + " seq_no=" + op.seqNo() + " op_term=" + op.primaryTerm() + " existing_term=" + versionValue.term; status = OpVsLuceneDocStatus.OP_STALE_OR_EQUAL; } else { final DocIdAndSeqNo docAndSeqNo = VersionsAndSeqNoResolver.loadDocIdAndSeqNo(searcher.reader(), op.uid()); if (docAndSeqNo == null) { status = OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND; } else if (op.seqNo() > docAndSeqNo.seqNo) { if (docAndSeqNo.isLive) { status = OpVsLuceneDocStatus.OP_NEWER; status = OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND; } else if (op.seqNo() == docAndSeqNo.seqNo) { assert localCheckpointTracker.contains(op.seqNo()) || softDeleteEnabled == false : "local checkpoint tracker is not updated seq_no=" + op.seqNo() + " id=" + op.id(); status = OpVsLuceneDocStatus.OP_STALE_OR_EQUAL; } else {
private void ensureWriteAllowed(Engine.Operation op) throws IllegalIndexShardStateException { Engine.Operation.Origin origin = op.origin(); IndexShardState state = this.state; // one time volatile read if (origin == Engine.Operation.Origin.PRIMARY) { // for primaries, we only allow to write when actually started (so the cluster has decided we started) // otherwise, we need to retry, we also want to still allow to index if we are relocated in case it fails if (state != IndexShardState.STARTED && state != IndexShardState.RELOCATED) { throw new IllegalIndexShardStateException(shardId, state, "operation only allowed when started/recovering, origin [" + origin + "]"); } } else { // for replicas, we allow to write also while recovering, since we index also during recovery to replicas // and rely on version checks to make sure its consistent if (state != IndexShardState.STARTED && state != IndexShardState.RELOCATED && state != IndexShardState.RECOVERING && state != IndexShardState.POST_RECOVERY) { throw new IllegalIndexShardStateException(shardId, state, "operation only allowed when started/recovering, origin [" + origin + "]"); } } }
/** resolves the current version of the document, returning null if not found */ private VersionValue resolveDocVersion(final Operation op) throws IOException { assert incrementVersionLookup(); // used for asserting in tests //VersionValue versionValue = versionMap.getUnderLock(op.uid()); VersionValue versionValue = null; if (versionValue == null) { assert incrementIndexVersionLookup(); // used for asserting in tests final long currentVersion = loadCurrentVersionFromIndex(op.uid()); if (currentVersion != Versions.NOT_FOUND) { versionValue = new VersionValue(currentVersion); } } else if (engineConfig.isEnableGcDeletes() && versionValue.isDelete() && (engineConfig.getThreadPool().relativeTimeInMillis() - versionValue.getTime()) > getGcDeletesInMillis()) { versionValue = null; } return versionValue; }
/** resolves the current version of the document, returning null if not found */ private VersionValue resolveDocVersion(final Operation op) throws IOException { assert incrementVersionLookup(); // used for asserting in tests VersionValue versionValue = versionMap.getUnderLock(op.uid()); if (versionValue == null) { assert incrementIndexVersionLookup(); // used for asserting in tests final long currentVersion = loadCurrentVersionFromIndex(op.uid()); if (currentVersion != Versions.NOT_FOUND) { versionValue = new VersionValue(currentVersion); } } else if (engineConfig.isEnableGcDeletes() && versionValue.isDelete() && (engineConfig.getThreadPool().relativeTimeInMillis() - versionValue.getTime()) > getGcDeletesInMillis()) { versionValue = null; } return versionValue; }
/** resolves the current version of the document, returning null if not found */ private VersionValue resolveDocVersion(final Operation op) throws IOException { assert incrementVersionLookup(); // used for asserting in tests VersionValue versionValue = getVersionFromMap(op.uid().bytes()); if (versionValue == null) { assert incrementIndexVersionLookup(); // used for asserting in tests final long currentVersion = loadCurrentVersionFromIndex(op.uid()); if (currentVersion != Versions.NOT_FOUND) { versionValue = new IndexVersionValue(null, currentVersion, SequenceNumbers.UNASSIGNED_SEQ_NO, 0L); } } else if (engineConfig.isEnableGcDeletes() && versionValue.isDelete() && (engineConfig.getThreadPool().relativeTimeInMillis() - ((DeleteVersionValue)versionValue).time) > getGcDeletesInMillis()) { versionValue = null; } return versionValue; }
/** resolves the current version of the document, returning null if not found */ private VersionValue resolveDocVersion(final Operation op) throws IOException { assert incrementVersionLookup(); // used for asserting in tests VersionValue versionValue = getVersionFromMap(op.uid().bytes()); if (versionValue == null) { assert incrementIndexVersionLookup(); // used for asserting in tests final long currentVersion = loadCurrentVersionFromIndex(op.uid()); if (currentVersion != Versions.NOT_FOUND) { versionValue = new IndexVersionValue(null, currentVersion, SequenceNumbers.UNASSIGNED_SEQ_NO, 0L); } } else if (engineConfig.isEnableGcDeletes() && versionValue.isDelete() && (engineConfig.getThreadPool().relativeTimeInMillis() - ((DeleteVersionValue)versionValue).time) > getGcDeletesInMillis()) { versionValue = null; } return versionValue; }
private boolean assertVersionType(final Engine.Operation operation) { if (operation.origin() == Operation.Origin.REPLICA || operation.origin() == Operation.Origin.PEER_RECOVERY || operation.origin() == Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { // ensure that replica operation has expected version type for replication // ensure that versionTypeForReplicationAndRecovery is idempotent assert operation.versionType() == operation.versionType().versionTypeForReplicationAndRecovery() : "unexpected version type in request from [" + operation.origin().name() + "] " + "found [" + operation.versionType().name() + "] " + "expected [" + operation.versionType().versionTypeForReplicationAndRecovery().name() + "]"; } return true; }
/** * Checks if the given operation has been processed in this engine or not. * @return true if the given operation was processed; otherwise false. */ protected final boolean hasBeenProcessedBefore(Operation op) { if (Assertions.ENABLED) { assert op.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "operation is not assigned seq_no"; if (op.operationType() == Operation.TYPE.NO_OP) { assert noOpKeyedLock.isHeldByCurrentThread(op.seqNo()); } else { assert versionMap.assertKeyedLockHeldByCurrentThread(op.uid().bytes()); } } return localCheckpointTracker.contains(op.seqNo()); }
public VersionConflictEngineException(ShardId shardId, Engine.Operation op, long currentVersion, boolean deleted) { this(shardId, op.type(), op.id(), op.versionType().explainConflictForWrites(currentVersion, op.version(), deleted)); }
public VersionConflictEngineException(ShardId shardId, Engine.Operation op, long currentVersion, boolean deleted) { this(shardId, op.type(), op.id(), op.versionType().explainConflictForWrites(currentVersion, op.version(), deleted)); }
public VersionConflictEngineException(ShardId shardId, Engine.Operation op, long currentVersion, boolean deleted) { this(shardId, op.type(), op.id(), op.versionType().explainConflictForWrites(currentVersion, op.version(), deleted)); }
private boolean assertVersionType(final Engine.Operation operation) { if (operation.origin() == Operation.Origin.REPLICA || operation.origin() == Operation.Origin.PEER_RECOVERY || operation.origin() == Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { // ensure that replica operation has expected version type for replication // ensure that versionTypeForReplicationAndRecovery is idempotent assert operation.versionType() == operation.versionType().versionTypeForReplicationAndRecovery() : "unexpected version type in request from [" + operation.origin().name() + "] " + "found [" + operation.versionType().name() + "] " + "expected [" + operation.versionType().versionTypeForReplicationAndRecovery().name() + "]"; } return true; }
private boolean assertVersionType(final Engine.Operation operation) { if (operation.origin() == Operation.Origin.REPLICA || operation.origin() == Operation.Origin.PEER_RECOVERY || operation.origin() == Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { // ensure that replica operation has expected version type for replication // ensure that versionTypeForReplicationAndRecovery is idempotent assert operation.versionType() == operation.versionType().versionTypeForReplicationAndRecovery() : "unexpected version type in request from [" + operation.origin().name() + "] " + "found [" + operation.versionType().name() + "] " + "expected [" + operation.versionType().versionTypeForReplicationAndRecovery().name() + "]"; } return true; }
private boolean assertVersionType(final Engine.Operation operation) { if (operation.origin() == Operation.Origin.REPLICA || operation.origin() == Operation.Origin.PEER_RECOVERY || operation.origin() == Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { // ensure that replica operation has expected version type for replication // ensure that versionTypeForReplicationAndRecovery is idempotent assert operation.versionType() == operation.versionType().versionTypeForReplicationAndRecovery() : "unexpected version type in request from [" + operation.origin().name() + "] " + "found [" + operation.versionType().name() + "] " + "expected [" + operation.versionType().versionTypeForReplicationAndRecovery().name() + "]"; } return true; }
/** called by IndexShard to record estimated bytes written to translog for the operation */ private void recordOperationBytes(Engine.Operation operation, Engine.Result result) { if (result.getResultType() == Engine.Result.Type.SUCCESS) { statusChecker.bytesWritten(operation.estimatedSizeInBytes()); } }
private OpVsLuceneDocStatus compareOpToLuceneDocBasedOnVersions(final Operation op) throws IOException { assert op.seqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO : "op is resolved based on versions but have a seq#"; assert op.version() >= 0 : "versions should be non-negative. got " + op.version(); final VersionValue versionValue = resolveDocVersion(op); if (versionValue == null) { return OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND; } else { return op.versionType().isVersionConflictForWrites(versionValue.version, op.version(), versionValue.isDelete()) ? OpVsLuceneDocStatus.OP_STALE_OR_EQUAL : OpVsLuceneDocStatus.OP_NEWER; } }