protected Engine.Index replicaIndexForDoc(ParsedDocument doc, long version, long seqNo, boolean isRetry) { return new Engine.Index(newUid(doc), doc, seqNo, 1, version, VersionType.EXTERNAL, Engine.Operation.Origin.REPLICA, System.nanoTime(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, isRetry); }
@Override public void postIndex(ShardId shardId, Engine.Index indexOperation, Engine.IndexResult result) { if (result.getResultType() == Engine.Result.Type.SUCCESS) { final ParsedDocument doc = indexOperation.parsedDoc(); final long tookInNanos = result.getTook(); if (indexWarnThreshold >= 0 && tookInNanos > indexWarnThreshold) { indexLogger.warn("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog)); } else if (indexInfoThreshold >= 0 && tookInNanos > indexInfoThreshold) { indexLogger.info("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog)); } else if (indexDebugThreshold >= 0 && tookInNanos > indexDebugThreshold) { indexLogger.debug("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog)); } else if (indexTraceThreshold >= 0 && tookInNanos > indexTraceThreshold) { indexLogger.trace("{}", new SlowLogParsedDocumentPrinter(index, doc, tookInNanos, reformat, maxSourceCharsToLog)); } } }
final Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); final boolean dynamicMappingEnable = indexService.mapperService().dynamic(); if (update != null && dynamicMappingEnable) {
this.id = index.id(); this.type = index.type(); this.source = index.source(); this.routing = index.routing(); this.parent = index.parent(); this.seqNo = indexResult.getSeqNo(); this.primaryTerm = index.primaryTerm(); this.version = indexResult.getVersion(); this.versionType = index.versionType(); this.autoGeneratedIdTimestamp = index.getAutoGeneratedIdTimestamp();
plan.versionForIndexing, getPrimaryTerm(), plan.seqNoForIndexing, plan.currentNotFoundOrDeleted); if (index.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { final Translog.Location location; if (indexResult.getResultType() == Result.Type.SUCCESS) { versionMap.maybePutIndexUnderLock(index.uid().bytes(), new IndexVersionValue(translogLocation, plan.versionForIndexing, plan.seqNoForIndexing, index.primaryTerm())); localCheckpointTracker.markSeqNoAsCompleted(indexResult.getSeqNo()); indexResult.setTook(System.nanoTime() - index.startTime()); indexResult.freeze(); return indexResult;
@Override public IndexResult index(Index index) throws IOException { assert Objects.equals(index.uid().field(), uidField) : index.uid().field(); final boolean doThrottle = index.origin().isRecovery() == false; try (ReleasableLock releasableLock = readLock.acquire()) { ensureOpen(); assert assertIncomingSequenceNumber(index.origin(), index.seqNo()); assert assertVersionType(index); try (Releasable ignored = versionMap.acquireLock(index.uid().bytes()); Releasable indexThrottle = doThrottle ? () -> {} : throttle.acquireThrottle()) { lastWriteNanos = index.startTime(); plan.versionForIndexing, getPrimaryTerm(), plan.seqNoForIndexing, plan.currentNotFoundOrDeleted); if (index.origin().isFromTranslog() == false) { final Translog.Location location; if (indexResult.getResultType() == Result.Type.SUCCESS) { final NoOp noOp = new NoOp(indexResult.getSeqNo(), index.primaryTerm(), index.origin(), index.startTime(), indexResult.getFailure().toString()); location = innerNoOp(noOp).getTranslogLocation(); } else { versionMap.maybePutIndexUnderLock(index.uid().bytes(), new IndexVersionValue(translogLocation, plan.versionForIndexing, plan.seqNoForIndexing, index.primaryTerm())); indexResult.setTook(System.nanoTime() - index.startTime()); indexResult.freeze(); return indexResult;
@Override public IndexResult index(Index index) throws IOException { assert Objects.equals(index.uid().field(), uidField) : index.uid().field(); final boolean doThrottle = index.origin().isRecovery() == false; try (ReleasableLock releasableLock = readLock.acquire()) { ensureOpen(); assert assertVersionType(index); try (Releasable ignored = acquireLock(index.uid()); Releasable indexThrottle = doThrottle ? () -> {} : throttle.acquireThrottle()) { lastWriteNanos = index.startTime(); if (index.origin() == Operation.Origin.PRIMARY) { plan = planIndexingAsPrimary(index); } else { index.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { Translog.Location location = translog.add(new Translog.Index(index, indexResult)); indexResult.setTranslogLocation(location); indexResult.setTook(System.nanoTime() - index.startTime()); indexResult.freeze(); return indexResult;
.routing(index.routing()).parent(index.parent()).timestamp(index.timestamp()).ttl(index.ttl()), index.version(), index.versionType().versionTypeForReplicationAndRecovery(), Engine.Operation.Origin.RECOVERY, true); maybeAddMappingUpdate(engineIndex.type(), engineIndex.parsedDoc().dynamicMappingsUpdate(), engineIndex.id(), allowMappingUpdates); if (logger.isTraceEnabled()) { logger.trace("[translog] recover [index] op of [{}][{}]", index.type(), index.id());
@Override public IndexResult index(Index index) throws IOException { assert Objects.equals(index.uid().field(), uidField) : index.uid().field(); final boolean doThrottle = index.origin().isRecovery() == false; try (ReleasableLock releasableLock = readLock.acquire()) { ensureOpen(); assert assertVersionType(index); try (Releasable ignored = acquireLock(index.uid()); Releasable indexThrottle = doThrottle ? () -> {} : throttle.acquireThrottle()) { lastWriteNanos = index.startTime(); if (index.origin() == Operation.Origin.PRIMARY) { plan = planIndexingAsPrimary(index); } else { index.origin() != Operation.Origin.LOCAL_TRANSLOG_RECOVERY) { indexResult.setTranslogLocation( translog.add(index.estimatedSizeInBytes()) ); indexResult.setTook(System.nanoTime() - index.startTime()); indexResult.freeze(); return indexResult;
final IndexingStrategy plan; final boolean appendOnlyRequest = canOptimizeAddDocument(index); if (appendOnlyRequest && mayHaveBeenIndexedBefore(index) == false && index.seqNo() > maxSeqNoOfNonAppendOnlyOperations.get()) { assert index.version() == 1L : "can optimize on replicas but incoming version is [" + index.version() + "]"; plan = IndexingStrategy.optimizedAppendOnly(index.seqNo(), 1L); } else { if (appendOnlyRequest == false) { maxSeqNoOfNonAppendOnlyOperations.updateAndGet(curr -> Math.max(index.seqNo(), curr)); assert maxSeqNoOfNonAppendOnlyOperations.get() >= index.seqNo() : "max_seqno of non-append-only was not updated;" + "max_seqno non-append-only [" + maxSeqNoOfNonAppendOnlyOperations.get() + "], seqno of index [" + index.seqNo() + "]"; assert index.versionType().versionTypeForReplicationAndRecovery() == index.versionType() : "resolving out of order delivery based on versioning but version type isn't fit for it. got [" + index.versionType() + "]"; if (index.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO && index.seqNo() <= localCheckpointTracker.getCheckpoint()){ plan = IndexingStrategy.processButSkipLucene(false, index.seqNo(), index.version()); } else { final OpVsLuceneDocStatus opVsLucene; if (index.seqNo() == SequenceNumbers.UNASSIGNED_SEQ_NO) { plan = IndexingStrategy.processAsStaleOp(softDeleteEnabled, index.seqNo(), index.version()); } else { plan = IndexingStrategy.processNormally(opVsLucene == OpVsLuceneDocStatus.LUCENE_DOC_NOT_FOUND, index.seqNo(), index.version());
baseCfs.metadata.ksName, baseCfs.metadata.cfName, context.indexInfo.name, typeName, parsedDoc.id(), operation.version(), result.isCreated(), isStatic(), ttl, context.indexInfo.refresh);
.routing(index.routing()).parent(index.parent()).timestamp(index.timestamp()).ttl(index.ttl()), index.version(), index.versionType().versionTypeForReplicationAndRecovery(), origin, index.getAutoGeneratedIdTimestamp(), true); maybeAddMappingUpdate(engineIndex.type(), engineIndex.parsedDoc().dynamicMappingsUpdate(), engineIndex.id(), allowMappingUpdates); if (logger.isTraceEnabled()) { logger.trace("[translog] recover [index] op of [{}][{}]", index.type(), index.id());
private boolean innerIndex(Index index) throws IOException { try (Releasable ignored = acquireLock(index.uid())) { final long currentVersion; VersionValue versionValue = versionMap.getUnderLock(index.uid().bytes()); if (versionValue == null) { currentVersion = loadCurrentVersionFromIndex(index.uid()); } else { if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis() - long expectedVersion = index.version(); if (index.versionType().isVersionConflictForWrites(currentVersion, expectedVersion)) { if (index.origin() == Operation.Origin.RECOVERY) { return false; } else { throw new VersionConflictEngineException(shardId, index.type(), index.id(), currentVersion, expectedVersion); updatedVersion = index.versionType().updateVersion(currentVersion, expectedVersion); index.updateVersion(updatedVersion); if (currentVersion == Versions.NOT_FOUND) { if (index.docs().size() > 1) { indexWriter.addDocuments(index.docs()); } else { indexWriter.addDocument(index.docs().get(0)); if (index.docs().size() > 1) { indexWriter.updateDocuments(index.uid(), index.docs()); } else { indexWriter.updateDocument(index.uid(), index.docs().get(0));
index.parsedDoc().version().setLongValue(plan.versionForIndexing); try { if (plan.useLuceneUpdateDocument) {
protected final IndexingStrategy planIndexingAsPrimary(Index index) throws IOException { assert index.origin() == Operation.Origin.PRIMARY : "planing as primary but origin isn't. got " + index.origin(); assert getMaxSeqNoOfUpdatesOrDeletes() != SequenceNumbers.UNASSIGNED_SEQ_NO : "max_seq_no_of_updates is not initialized"; final IndexingStrategy plan; currentNotFoundOrDeleted = versionValue.isDelete(); if (index.versionType().isVersionConflictForWrites( currentVersion, index.version(), currentNotFoundOrDeleted)) { final VersionConflictEngineException e = new VersionConflictEngineException(shardId, index, currentVersion, currentNotFoundOrDeleted); plan = IndexingStrategy.processNormally(currentNotFoundOrDeleted, generateSeqNoForOperation(index), index.versionType().updateVersion(currentVersion, index.version()) );
private IndexResult indexIntoLucene(Index index, IndexingStrategy plan) throws IOException { assert assertSequenceNumberBeforeIndexing(index.origin(), plan.seqNoForIndexing); assert plan.versionForIndexing >= 0 : "version must be set. got " + plan.versionForIndexing; assert plan.indexIntoLucene || plan.addStaleOpToLucene; index.parsedDoc().updateSeqID(plan.seqNoForIndexing, index.primaryTerm()); index.parsedDoc().version().setLongValue(plan.versionForIndexing); try { if (plan.addStaleOpToLucene) { addStaleDocs(index.docs(), indexWriter); } else if (plan.useLuceneUpdateDocument) { assert assertMaxSeqNoOfUpdatesIsAdvanced(index.uid(), plan.seqNoForIndexing, true, true); updateDocs(index.uid(), index.docs(), indexWriter); } else { addDocs(index.docs(), indexWriter);
currentNotFoundOrDeleted = versionValue.isDelete(); if (index.versionType().isVersionConflictForWrites( currentVersion, index.version(), currentNotFoundOrDeleted)) { final VersionConflictEngineException e = new VersionConflictEngineException(shardId, index, currentVersion, currentNotFoundOrDeleted); plan = IndexingStrategy.processNormally(currentNotFoundOrDeleted, generateSeqNoForOperation(index), index.versionType().updateVersion(currentVersion, index.version()) );
return new Engine.IndexResult(e, request.version()); Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); final ShardId shardId = primary.shardId(); if (update != null) { return new Engine.IndexResult(e, request.version()); update = operation.parsedDoc().dynamicMappingsUpdate(); if (update != null) { throw new ReplicationOperation.RetryOnPrimaryException(shardId,