beforeMerge(onGoingMerge); super.doMerge(writer, merge); } finally { afterMerge(onGoingMerge);
void refreshConfig() { if (this.getMaxMergeCount() != config.getMaxMergeCount() || this.getMaxThreadCount() != config.getMaxThreadCount()) { this.setMaxMergesAndThreads(config.getMaxMergeCount(), config.getMaxThreadCount()); } boolean isEnabled = getIORateLimitMBPerSec() != Double.POSITIVE_INFINITY; if (config.isAutoThrottle() && isEnabled == false) { enableAutoIOThrottle(); } else if (config.isAutoThrottle() == false && isEnabled) { disableAutoIOThrottle(); } }
@Override public List<Segment> segments(boolean verbose) { try (ReleasableLock lock = readLock.acquire()) { Segment[] segmentsArr = getSegmentInfo(lastCommittedSegmentInfos, verbose); // fill in the merges flag Set<OnGoingMerge> onGoingMerges = mergeScheduler.onGoingMerges(); for (OnGoingMerge onGoingMerge : onGoingMerges) { for (SegmentCommitInfo segmentInfoPerCommit : onGoingMerge.getMergedSegments()) { for (Segment segment : segmentsArr) { if (segment.getName().equals(segmentInfoPerCommit.info.name)) { segment.mergeId = onGoingMerge.getId(); break; } } } } return Arrays.asList(segmentsArr); } }
ElasticsearchConcurrentMergeScheduler(ShardId shardId, IndexSettings indexSettings) { this.config = indexSettings.getMergeSchedulerConfig(); this.shardId = shardId; this.indexSettings = indexSettings.getSettings(); this.logger = Loggers.getLogger(getClass(), shardId); refreshConfig(); }
@Override public synchronized void beforeMerge(OnGoingMerge merge) { int maxNumMerges = mergeScheduler.getMaxMergeCount(); if (numMergesInFlight.incrementAndGet() > maxNumMerges) { if (isThrottling.getAndSet(true) == false) { logger.info("now throttling indexing: numMergesInFlight={}, maxNumMerges={}", numMergesInFlight, maxNumMerges); activateThrottling(); } } }
@Override public double getIORateLimitMBPerSec() { return this.mergeScheduler.getIORateLimitMBPerSec(); }
public void onSettingsChanged() { mergeScheduler.refreshConfig(); // config().isEnableGcDeletes() or config.getGcDeletesInMillis() may have changed: maybePruneDeletes(); if (engineConfig.isAutoGeneratedIDsOptimizationEnabled() == false) { // this is an anti-viral settings you can only opt out for the entire index // only if a shard starts up again due to relocation or if the index is closed // the setting will be re-interpreted if it's set to true updateAutoIdTimestamp(Long.MAX_VALUE, true); } final TranslogDeletionPolicy translogDeletionPolicy = translog.getDeletionPolicy(); final IndexSettings indexSettings = engineConfig.getIndexSettings(); translogDeletionPolicy.setRetentionAgeInMillis(indexSettings.getTranslogRetentionAge().getMillis()); translogDeletionPolicy.setRetentionSizeInBytes(indexSettings.getTranslogRetentionSize().getBytes()); softDeletesPolicy.setRetentionOperations(indexSettings.getSoftDeleteRetentionOperations()); }
@Override public synchronized void afterMerge(OnGoingMerge merge) { int maxNumMerges = mergeScheduler.getMaxMergeCount(); if (numMergesInFlight.decrementAndGet() < maxNumMerges) { if (isThrottling.getAndSet(false)) {
MergeStats stats() { final MergeStats mergeStats = new MergeStats(); mergeStats.add(totalMerges.count(), totalMerges.sum(), totalMergesNumDocs.count(), totalMergesSizeInBytes.count(), currentMerges.count(), currentMergesNumDocs.count(), currentMergesSizeInBytes.count(), totalMergeStoppedTime.count(), totalMergeThrottledTime.count(), config.isAutoThrottle() ? getIORateLimitMBPerSec() : Double.POSITIVE_INFINITY); return mergeStats; }
void refreshConfig() { if (this.getMaxMergeCount() != config.getMaxMergeCount() || this.getMaxThreadCount() != config.getMaxThreadCount()) { this.setMaxMergesAndThreads(config.getMaxMergeCount(), config.getMaxThreadCount()); } boolean isEnabled = getIORateLimitMBPerSec() != Double.POSITIVE_INFINITY; if (config.isAutoThrottle() && isEnabled == false) { enableAutoIOThrottle(); } else if (config.isAutoThrottle() == false && isEnabled) { disableAutoIOThrottle(); } }
beforeMerge(onGoingMerge); super.doMerge(writer, merge); } finally { afterMerge(onGoingMerge);
mergeScheduler.refreshConfig();
@Override public synchronized void beforeMerge(OnGoingMerge merge) { int maxNumMerges = mergeScheduler.getMaxMergeCount(); if (numMergesInFlight.incrementAndGet() > maxNumMerges) { if (isThrottling.getAndSet(true) == false) { logger.info("now throttling indexing: numMergesInFlight={}, maxNumMerges={}", numMergesInFlight, maxNumMerges); activateThrottling(); } } }
MergeStats stats() { final MergeStats mergeStats = new MergeStats(); mergeStats.add(totalMerges.count(), totalMerges.sum(), totalMergesNumDocs.count(), totalMergesSizeInBytes.count(), currentMerges.count(), currentMergesNumDocs.count(), currentMergesSizeInBytes.count(), totalMergeStoppedTime.count(), totalMergeThrottledTime.count(), config.isAutoThrottle() ? getIORateLimitMBPerSec() : Double.POSITIVE_INFINITY); return mergeStats; }
@Override public List<Segment> segments(boolean verbose) { try (ReleasableLock lock = readLock.acquire()) { Segment[] segmentsArr = getSegmentInfo(lastCommittedSegmentInfos, verbose); // fill in the merges flag Set<OnGoingMerge> onGoingMerges = mergeScheduler.onGoingMerges(); for (OnGoingMerge onGoingMerge : onGoingMerges) { for (SegmentCommitInfo segmentInfoPerCommit : onGoingMerge.getMergedSegments()) { for (Segment segment : segmentsArr) { if (segment.getName().equals(segmentInfoPerCommit.info.name)) { segment.mergeId = onGoingMerge.getId(); break; } } } } return Arrays.asList(segmentsArr); } }
void refreshConfig() { if (this.getMaxMergeCount() != config.getMaxMergeCount() || this.getMaxThreadCount() != config.getMaxThreadCount()) { this.setMaxMergesAndThreads(config.getMaxMergeCount(), config.getMaxThreadCount()); } boolean isEnabled = getIORateLimitMBPerSec() != Double.POSITIVE_INFINITY; if (config.isAutoThrottle() && isEnabled == false) { enableAutoIOThrottle(); } else if (config.isAutoThrottle() == false && isEnabled) { disableAutoIOThrottle(); } }
beforeMerge(onGoingMerge); super.doMerge(writer, merge); } finally { afterMerge(onGoingMerge);
public ElasticsearchConcurrentMergeScheduler(ShardId shardId, Settings indexSettings, MergeSchedulerConfig config) { this.config = config; this.shardId = shardId; this.indexSettings = indexSettings; this.logger = Loggers.getLogger(getClass(), indexSettings, shardId); refreshConfig(); }
@Override public synchronized void beforeMerge(OnGoingMerge merge) { int maxNumMerges = mergeScheduler.getMaxMergeCount(); if (numMergesInFlight.incrementAndGet() > maxNumMerges) { if (isThrottling.getAndSet(true) == false) { logger.info("now throttling indexing: numMergesInFlight={}, maxNumMerges={}", numMergesInFlight, maxNumMerges); activateThrottling(); } } }
MergeStats stats() { final MergeStats mergeStats = new MergeStats(); mergeStats.add(totalMerges.count(), totalMerges.sum(), totalMergesNumDocs.count(), totalMergesSizeInBytes.count(), currentMerges.count(), currentMergesNumDocs.count(), currentMergesSizeInBytes.count(), totalMergeStoppedTime.count(), totalMergeThrottledTime.count(), config.isAutoThrottle() ? getIORateLimitMBPerSec() : Double.POSITIVE_INFINITY); return mergeStats; }