public CommitPoint(Collection<CommitPoint> commitsToDelete, Directory directoryOrig, SegmentInfos segmentInfos) throws IOException { this.directoryOrig = directoryOrig; this.commitsToDelete = commitsToDelete; userData = segmentInfos.getUserData(); segmentsFileName = segmentInfos.getSegmentsFileName(); generation = segmentInfos.getGeneration(); files = Collections.unmodifiableCollection(segmentInfos.files(true)); segmentCount = segmentInfos.size(); }
ReaderCommit(StandardDirectoryReader reader, SegmentInfos infos, Directory dir) throws IOException { segmentsFileName = infos.getSegmentsFileName(); this.dir = dir; userData = infos.getUserData(); files = Collections.unmodifiableCollection(infos.files(true)); generation = infos.getGeneration(); segmentCount = infos.size(); // NOTE: we intentionally do not incRef this! Else we'd need to make IndexCommit Closeable... this.reader = reader; }
result.segmentsFileName = segmentsFileName; result.numSegments = numSegments; result.userData = sis.getUserData(); String userDataString; if (sis.getUserData().size() > 0) { userDataString = " userData=" + sis.getUserData(); } else { userDataString = "";
/** * Reads the current stored translog ID from the last commit data. */ @Nullable private String loadTranslogUUIDFromLastCommit() throws IOException { final Map<String, String> commitUserData = store.readLastCommittedSegmentsInfo().getUserData(); if (commitUserData.containsKey(Translog.TRANSLOG_GENERATION_KEY) == false) { throw new IllegalStateException("commit doesn't contain translog generation id"); } return commitUserData.get(Translog.TRANSLOG_UUID_KEY); }
/** * Marks an existing lucene index with a new history uuid. * This is used to make sure no existing shard will recovery from this index using ops based recovery. */ public void bootstrapNewHistory() throws IOException { metadataLock.writeLock().lock(); try { Map<String, String> userData = readLastCommittedSegmentsInfo().getUserData(); final SequenceNumbers.CommitInfo seqno = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(userData.entrySet()); bootstrapNewHistory(seqno.maxSeqNo); } finally { metadataLock.writeLock().unlock(); } }
commitUserData = new HashMap<>(segmentInfos.getUserData()).entrySet();
private boolean assertSequenceNumbersInCommit() throws IOException { final Map<String, String> userData = SegmentInfos.readLatestCommit(store.directory()).getUserData(); assert userData.containsKey(SequenceNumbers.LOCAL_CHECKPOINT_KEY) : "commit point doesn't contains a local checkpoint"; assert userData.containsKey(SequenceNumbers.MAX_SEQ_NO) : "commit point doesn't contains a maximum sequence number"; assert userData.containsKey(Engine.HISTORY_UUID_KEY) : "commit point doesn't contains a history uuid"; assert userData.get(Engine.HISTORY_UUID_KEY).equals(getHistoryUUID()) : "commit point history uuid [" + userData.get(Engine.HISTORY_UUID_KEY) + "] is different than engine [" + getHistoryUUID() + "]"; return true; }
private boolean assertMaxUnsafeAutoIdInCommit() throws IOException { final Map<String, String> userData = SegmentInfos.readLatestCommit(store.directory()).getUserData(); if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_5_5_0)) { // as of 5.5.0, the engine stores the maxUnsafeAutoIdTimestamp in the commit point. // This should have baked into the commit by the primary we recover from, regardless of the index age. assert userData.containsKey(Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID) : "opening index which was created post 5.5.0 but " + Engine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID + " is not found in commit"; } return true; }
private CommitPoint(SegmentInfos infos, Directory dir) throws IOException { segmentsFileName = infos.getSegmentsFileName(); this.dir = dir; userData = infos.getUserData(); files = Collections.unmodifiableCollection(infos.files(true)); generation = infos.getGeneration(); segmentCount = infos.size(); }
public CommitStats(SegmentInfos segmentInfos) { // clone the map to protect against concurrent changes userData = MapBuilder.<String, String>newMapBuilder().putAll(segmentInfos.getUserData()).immutableMap(); // lucene calls the current generation, last generation. generation = segmentInfos.getLastGeneration(); id = Base64.getEncoder().encodeToString(segmentInfos.getId()); numDocs = Lucene.getNumDocs(segmentInfos); }
private void trimUnsafeCommits() throws IOException { assert currentEngineReference.get() == null || currentEngineReference.get() instanceof ReadOnlyEngine : "a write engine is running"; final String translogUUID = store.readLastCommittedSegmentsInfo().getUserData().get(Translog.TRANSLOG_UUID_KEY); final long globalCheckpoint = Translog.readGlobalCheckpoint(translogConfig.getTranslogPath(), translogUUID); final long minRetainedTranslogGen = Translog.readMinTranslogGeneration(translogConfig.getTranslogPath(), translogUUID); assertMaxUnsafeAutoIdInCommit(); store.trimUnsafeCommits(globalCheckpoint, minRetainedTranslogGen, indexSettings.getIndexVersionCreated()); }
final boolean tryRenewSyncCommit() { boolean renewed = false; try (ReleasableLock lock = writeLock.acquire()) { ensureOpen(); ensureCanFlush(); String syncId = lastCommittedSegmentInfos.getUserData().get(SYNC_COMMIT_ID); long translogGenOfLastCommit = Long.parseLong(lastCommittedSegmentInfos.userData.get(Translog.TRANSLOG_GENERATION_KEY)); if (syncId != null && indexWriter.hasUncommittedChanges() && translog.totalOperationsByMinGen(translogGenOfLastCommit) == 0) { logger.trace("start renewing sync commit [{}]", syncId); commitIndexWriter(indexWriter, translog, syncId); logger.debug("successfully sync committed. sync id [{}].", syncId); lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo(); renewed = true; } } catch (IOException ex) { maybeFailEngine("renew sync commit", ex); throw new EngineException(shardId, "failed to renew sync commit", ex); } if (renewed) { // refresh outside of the write lock // we have to refresh internal searcher here to ensure we release unreferenced segments. refresh("renew sync commit", SearcherScope.INTERNAL); } return renewed; }
private void recoverFromTranslogInternal(TranslogRecoveryRunner translogRecoveryRunner, long recoverUpToSeqNo) throws IOException { Translog.TranslogGeneration translogGeneration = translog.getGeneration(); final int opsRecovered; final long translogFileGen = Long.parseLong(lastCommittedSegmentInfos.getUserData().get(Translog.TRANSLOG_GENERATION_KEY)); try (Translog.Snapshot snapshot = translog.newSnapshotFromGen( new Translog.TranslogGeneration(translog.getTranslogUUID(), translogFileGen), recoverUpToSeqNo)) { opsRecovered = translogRecoveryRunner.run(this, snapshot); } catch (Exception e) { throw new EngineException(shardId, "failed to recover from translog", e); } // flush if we recovered something or if we have references to older translogs // note: if opsRecovered == 0 and we have older translogs it means they are corrupted or 0 length. assert pendingTranslogRecovery.get() : "translogRecovery is not pending but should be"; pendingTranslogRecovery.set(false); // we are good - now we can commit if (opsRecovered > 0) { logger.trace("flushing post recovery from translog. ops recovered [{}]. committed translog id [{}]. current id [{}]", opsRecovered, translogGeneration == null ? null : translogGeneration.translogFileGeneration, translog.currentFileGeneration()); commitIndexWriter(indexWriter, translog, null); refreshLastCommittedSegmentInfos(); refresh("translog_recovery"); } translog.trimUnreferencedReaders(); }
final SegmentInfos segmentCommitInfos = Store.readSegmentsInfo(commit, directory); numDocs = Lucene.getNumDocs(segmentCommitInfos); commitUserDataBuilder.putAll(segmentCommitInfos.getUserData());
try { final Store store = recoveryTarget.store(); final String translogUUID = store.readLastCommittedSegmentsInfo().getUserData().get(Translog.TRANSLOG_UUID_KEY); final long globalCheckpoint = Translog.readGlobalCheckpoint(recoveryTarget.translogLocation(), translogUUID); final List<IndexCommit> existingCommits = DirectoryReader.listCommits(store.directory());
final String translogUUID = store.readLastCommittedSegmentsInfo().getUserData().get(Translog.TRANSLOG_UUID_KEY); final long globalCheckpoint = Translog.readGlobalCheckpoint(translogConfig.getTranslogPath(), translogUUID); replicationTracker.updateGlobalCheckpointOnReplica(globalCheckpoint, "read from translog checkpoint");
/** * Reads the current stored translog ID from the last commit data. */ @Nullable private String loadTranslogUUIDFromLastCommit() throws IOException { final Map<String, String> commitUserData = store.readLastCommittedSegmentsInfo().getUserData(); if (commitUserData.containsKey(Translog.TRANSLOG_GENERATION_KEY) == false) { throw new IllegalStateException("commit doesn't contain translog generation id"); } return commitUserData.get(Translog.TRANSLOG_UUID_KEY); }
private CommitPoint(SegmentInfos infos, Directory dir) throws IOException { segmentsFileName = infos.getSegmentsFileName(); this.dir = dir; userData = infos.getUserData(); files = Collections.unmodifiableCollection(infos.files(dir, true)); generation = infos.getGeneration(); segmentCount = infos.size(); }
ReaderCommit(StandardDirectoryReader reader, SegmentInfos infos, Directory dir) throws IOException { segmentsFileName = infos.getSegmentsFileName(); this.dir = dir; userData = infos.getUserData(); files = Collections.unmodifiableCollection(infos.files(true)); generation = infos.getGeneration(); segmentCount = infos.size(); // NOTE: we intentionally do not incRef this! Else we'd need to make IndexCommit Closeable... this.reader = reader; }
public CommitStats(SegmentInfos segmentInfos) { // clone the map to protect against concurrent changes userData = MapBuilder.<String, String>newMapBuilder().putAll(segmentInfos.getUserData()).immutableMap(); // lucene calls the current generation, last generation. generation = segmentInfos.getLastGeneration(); id = Base64.getEncoder().encodeToString(segmentInfos.getId()); numDocs = Lucene.getNumDocs(segmentInfos); }