private InFlightOpsResponse performInFlightOps(InFlightOpsRequest request) { IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); IndexShard indexShard = indexService.getShard(request.shardId().id()); if (indexShard.routingEntry().primary() == false) { throw new IllegalStateException("[" + request.shardId() +"] expected a primary shard"); } int opCount = indexShard.getActiveOperationsCount(); return new InFlightOpsResponse(opCount); }
@Override public void onShardInactive(IndexShard indexShard) { for (IndexEventListener listener : listeners) { try { listener.onShardInactive(indexShard); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to invoke on shard inactive callback", indexShard.shardId().getId()), e); throw e; } } }
@Override public synchronized void removeShard(int shardId, String reason) { final ShardId sId = new ShardId(index(), shardId); final IndexShard indexShard; if (shards.containsKey(shardId) == false) { return; } logger.debug("[{}] closing... (reason: [{}])", shardId, reason); HashMap<Integer, IndexShard> newShards = new HashMap<>(shards); indexShard = newShards.remove(shardId); shards = unmodifiableMap(newShards); closeShard(reason, sId, indexShard, indexShard.store(), indexShard.getIndexEventListener()); logger.debug("[{}] closed (reason: [{}])", shardId, reason); }
@Override public void readFrom(StreamInput in) throws IOException { nodeId = in.readOptionalString(); shardId = ShardId.readShardId(in); super.shardId = shardId.getId(); index = shardId.getIndexName(); reason = in.readString(); status = RestStatus.readFrom(in); }
@Override public void readFrom(StreamInput in) throws IOException { shardId = ShardId.readShardId(in); super.shardId = shardId.getId(); index = shardId.getIndexName(); nodeId = in.readOptionalString(); cause = in.readException(); status = RestStatus.readFrom(in); primary = in.readBoolean(); }
public void processPendingDeletes(Index index, IndexSettings indexSettings, TimeValue timeout) throws IOException, InterruptedException, ShardLockObtainFailedException { logger.debug("{} processing pending deletes", index); final long startTimeNS = System.nanoTime(); final List<ShardLock> shardLocks = nodeEnv.lockAllForIndex(index, indexSettings, timeout.millis()); logger.debug("{} deleting index store reason [{}]", index, "pending delete"); try { nodeEnv.deleteIndexDirectoryUnderLock(index, indexSettings); iterator.remove(); } catch (IOException ex) { logger.debug(() -> new ParameterizedMessage("{} retry pending delete", index), ex); ShardLock shardLock = locks.get(new ShardId(delete.index, delete.shardId)); if (shardLock != null) { try { deleteShardStore("pending delete", shardLock, delete.settings); iterator.remove(); } catch (IOException ex) { logger.debug(() -> new ParameterizedMessage("{} retry pending delete", shardLock.getShardId()), ex);
throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed"); final ShardId shardId = routing.shardId(); boolean success = false; Store store = null; routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE if (shards.containsKey(shardId.id())) { throw new IllegalStateException(shardId + " already exists"); IndexShard shard = getShardOrNull(shardId.getId()); if (shard != null) { warmer.warm(searcher, shard, IndexService.this.indexSettings); store = new Store(shardId, this.indexSettings, directoryService.newDirectory(), lock, new StoreCloseListener(shardId, () -> eventListener.onStoreClosed(shardId))); indexShard = new IndexShard(routing, this.indexSettings, path, store, indexSortSupplier, indexCache, mapperService, similarityService, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer, searchOperationListeners, indexingOperationListeners, () -> globalCheckpointSyncer.accept(shardId), circuitBreakerService); eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); eventListener.afterIndexShardCreated(indexShard); shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap(); success = true; return indexShard;
@Override public void beforeIndexShardDeleted(ShardId shardId, Settings indexSettings) { for (IndexEventListener listener : listeners) { try { listener.beforeIndexShardDeleted(shardId, indexSettings); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("[{}] failed to invoke before shard deleted callback", shardId.getId()), e); throw e; } } }
final IndexMetaData metaData = clusterState.getMetaData().indices().get(shardId.getIndexName()); final IndexSettings indexSettings = buildIndexSettings(metaData); ShardDeletionCheckResult shardDeletionCheckResult = canDeleteShardContent(shardId, indexSettings); if (shardDeletionCheckResult != ShardDeletionCheckResult.FOLDER_FOUND_CAN_DELETE) { throw new IllegalStateException("Can't delete shard " + shardId + " (cause: " + shardDeletionCheckResult + ")"); logger.debug("{} deleted shard reason [{}]", shardId, reason); if (clusterState.nodes().getLocalNode().isMasterNode() == false && canDeleteIndexContents(shardId.getIndex(), indexSettings)) { if (nodeEnv.findAllShardIds(shardId.getIndex()).isEmpty()) { try { logger.trace("[{}] still has shard stores, leaving as is", shardId.getIndex());
private RecoveryResponse recover(final StartRecoveryRequest request) throws IOException { final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); final IndexShard shard = indexService.getShard(request.shardId().id()); final ShardRouting routingEntry = shard.routingEntry(); if (routingEntry.primary() == false || routingEntry.active() == false) { throw new DelayRecoveryException("source shard [" + routingEntry + "] is not an active primary"); } if (request.isPrimaryRelocation() && (routingEntry.relocating() == false || routingEntry.relocatingNodeId().equals(request.targetNode().getId()) == false)) { logger.debug("delaying recovery of {} as source shard is not marked yet as relocating to {}", request.shardId(), request.targetNode()); throw new DelayRecoveryException("source shard is not marked yet as relocating to [" + request.targetNode() + "]"); } RecoverySourceHandler handler = ongoingRecoveries.addNewRecovery(request, shard); logger.trace("[{}][{}] starting recovery to {}", request.shardId().getIndex().getName(), request.shardId().id(), request.targetNode()); try { return handler.recoverToTarget(); } finally { ongoingRecoveries.remove(shard, handler); } }
private StoreFilesMetaData listStoreMetaData(ShardId shardId) throws IOException { logger.trace("listing store meta data for {}", shardId); long startTimeNS = System.nanoTime(); boolean exists = false; try { IndexService indexService = indicesService.indexService(shardId.getIndex()); if (indexService != null) { IndexShard indexShard = indexService.getShardOrNull(shardId.id()); if (indexShard != null) { exists = true; return new StoreFilesMetaData(shardId, indexShard.snapshotStoreMetadata()); IndexMetaData metaData = clusterService.state().metaData().index(shardId.getIndex()); if (metaData == null) { nodeEnv.indexPaths(shardId.getIndex())); logger.trace("{} node doesn't have meta data for the requests index, responding with empty", shardId); return new StoreFilesMetaData(shardId, Store.MetadataSnapshot.EMPTY); TimeValue took = new TimeValue(System.nanoTime() - startTimeNS, TimeUnit.NANOSECONDS); if (exists) { logger.debug("{} loaded store meta data (took [{}])", shardId, took); } else { logger.trace("{} didn't find any store meta data to load (took [{}])", shardId, took);
try { final ShardId shardId = request.getShardId(); logger.trace("{} loading local shard state info", shardId); ShardStateMetaData shardStateMetaData = ShardStateMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.availableShardPaths(request.shardId)); if (shardStateMetaData != null) { IndexMetaData metaData = clusterService.state().metaData().index(shardId.getIndex()); if (metaData == null) { nodeEnv.indexPaths(shardId.getIndex())); if (indicesService.getShardOrNull(shardId) == null) { } catch (Exception exception) { final ShardPath finalShardPath = shardPath; logger.trace(() -> new ParameterizedMessage( "{} can't open index for shard [{}] in path [{}]", shardId, logger.debug("{} shard state info found: [{}]", shardId, shardStateMetaData); String allocationId = shardStateMetaData.allocationId != null ? shardStateMetaData.allocationId.getId() : null;
@Override public void accept(ShardLock lock) { try { assert lock.getShardId().equals(shardId) : "shard id mismatch, expected: " + shardId + " but got: " + lock.getShardId(); onShardClose(lock); } finally { try { IOUtils.close(toClose); } catch (IOException ex) { logger.debug("failed to close resource", ex); } } } }
final String localNodeId = state.nodes().getLocalNodeId(); assert localNodeId != null; RoutingNode localRoutingNode = state.getRoutingNodes().node(localNodeId); for (AllocatedIndex<? extends Shard> indexService : indicesService) { for (Shard shard : indexService) { ShardRouting currentRoutingEntry = shard.routingEntry(); ShardId shardId = currentRoutingEntry.shardId(); ShardRouting newShardRouting = localRoutingNode == null ? null : localRoutingNode.getByShardId(shardId); if (newShardRouting == null) { logger.debug("{} removing shard (not allocated)", shardId); indexService.removeShard(shardId.id(), "removing shard (not allocated)"); } else if (newShardRouting.isSameAllocation(currentRoutingEntry) == false) { logger.debug("{} removing shard (stale allocation id, stale {}, new {})", shardId, currentRoutingEntry, newShardRouting); indexService.removeShard(shardId.id(), "removing shard (stale copy)"); } else if (newShardRouting.initializing() && currentRoutingEntry.active()) { logger.debug("{} removing shard (not active, current {}, new {})", shardId, currentRoutingEntry, newShardRouting); indexService.removeShard(shardId.id(), "removing shard (stale copy)"); } else if (newShardRouting.primary() && currentRoutingEntry.primary() == false && newShardRouting.initializing()) { assert currentRoutingEntry.initializing() : currentRoutingEntry; // see above if clause logger.debug("{} removing shard (not active, current {}, new {})", shardId, currentRoutingEntry, newShardRouting); indexService.removeShard(shardId.id(), "removing shard (stale copy)");
@Override protected MultiGetShardResponse shardOperation(MultiGetShardRequest request, ShardId shardId) { IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.getShard(shardId.id()); if (request.refresh() && !request.realtime()) { indexShard.refresh("refresh_flag_mget"); } MultiGetShardResponse response = new MultiGetShardResponse(); for (int i = 0; i < request.locations.size(); i++) { MultiGetRequest.Item item = request.items.get(i); try { GetResult getResult = indexShard.getService().get(item.type(), item.id(), item.storedFields(), request.realtime(), item.version(), item.versionType(), item.fetchSourceContext()); response.add(request.locations.get(i), new GetResponse(getResult)); } catch (RuntimeException e) { if (TransportActions.isShardNotAvailableException(e)) { throw e; } else { logger.debug(() -> new ParameterizedMessage("{} failed to execute multi_get for [{}]/[{}]", shardId, item.type(), item.id()), e); response.add(request.locations.get(i), new MultiGetResponse.Failure(request.index(), item.type(), item.id(), e)); } } } return response; }
MetaData oldMetaData = clusterState.metaData(); RoutingTable oldRoutingTable = clusterState.routingTable(); MetaData.Builder metaDataBuilder = null; Collectors.groupingBy(fs -> fs.getShardId().getIndex())).entrySet()) { final IndexMetaData oldIndexMetaData = oldMetaData.getIndexSafe(indexEntry.getKey()); IndexMetaData.Builder indexMetaDataBuilder = null; int shardNumber = shardEntry.getKey().getId(); Set<String> oldInSyncAllocations = oldIndexMetaData.inSyncAllocationIds(shardNumber); Set<String> idsToRemove = shardEntry.getValue().stream().map(e -> e.getAllocationId()).collect(Collectors.toSet()); logger.warn("{} marking unavailable shards as stale: {}", shardEntry.getKey(), idsToRemove); return ClusterState.builder(clusterState).metaData(metaDataBuilder).build(); } else { return clusterState;
@Override protected MultiTermVectorsShardResponse shardOperation(MultiTermVectorsShardRequest request, ShardId shardId) { final MultiTermVectorsShardResponse response = new MultiTermVectorsShardResponse(); final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); final IndexShard indexShard = indexService.getShard(shardId.id()); for (int i = 0; i < request.locations.size(); i++) { TermVectorsRequest termVectorsRequest = request.requests.get(i); try { TermVectorsResponse termVectorsResponse = TermVectorsService.getTermVectors(indexShard, termVectorsRequest); response.add(request.locations.get(i), termVectorsResponse); } catch (RuntimeException e) { if (TransportActions.isShardNotAvailableException(e)) { throw e; } else { logger.debug(() -> new ParameterizedMessage("{} failed to execute multi term vectors for [{}]/[{}]", shardId, termVectorsRequest.type(), termVectorsRequest.id()), e); response.add(request.locations.get(i), new MultiTermVectorsResponse.Failure(request.index(), termVectorsRequest.type(), termVectorsRequest.id(), e)); } } } return response; }
Runnable globalCheckpointSyncer, CircuitBreakerService circuitBreakerService) throws IOException { super(shardRouting.shardId(), indexSettings); assert shardRouting.initializing(); this.shardRouting = shardRouting; final Settings settings = indexSettings.getSettings(); this.circuitBreakerService = circuitBreakerService; logger.debug("state: [CREATED]"); + "which has no effect and will not be accepted in future"); this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, bigArrays); final String aId = shardRouting.allocationId().getId(); this.globalCheckpointListeners = new GlobalCheckpointListeners(shardId, threadPool.executor(ThreadPool.Names.LISTENER), threadPool.scheduler(), logger); pendingPrimaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id()); operationPrimaryTerm = pendingPrimaryTerm; refreshListeners = buildRefreshListeners(); persistMetadata(path, indexSettings, shardRouting, null, logger);
private void onShardClose(ShardLock lock) { if (deleted.get()) { // we remove that shards content if this index has been deleted try { try { eventListener.beforeIndexShardDeleted(lock.getShardId(), indexSettings.getSettings()); } finally { shardStoreDeleter.deleteShardStore("delete index", lock, indexSettings); eventListener.afterIndexShardDeleted(lock.getShardId(), indexSettings.getSettings()); } } catch (IOException e) { shardStoreDeleter.addPendingDelete(lock.getShardId(), indexSettings); logger.debug( () -> new ParameterizedMessage( "[{}] failed to delete shard content - scheduled a retry", lock.getShardId().id()), e); } } }