if (request instanceof BulkShardRequest) { BulkShardRequest bsr = (BulkShardRequest) request; for (BulkItemRequest bir : bsr.items()) { requestTypes.add(bir.request().type());
/** builds the bulk shard response to return to the user */ public BulkShardResponse buildShardResponse() { assert hasMoreOperationsToExecute() == false; return new BulkShardResponse(request.shardId(), Arrays.stream(request.items()).map(BulkItemRequest::getPrimaryResponse).toArray(BulkItemResponse[]::new)); }
@Override public String toString() { // This is included in error messages so we'll try to make it somewhat user friendly. StringBuilder b = new StringBuilder("BulkShardRequest ["); b.append(shardId).append("] containing ["); if (items.length > 1) { b.append(items.length).append("] requests"); } else { b.append(items[0].request()).append("]"); } switch (getRefreshPolicy()) { case IMMEDIATE: b.append(" and a refresh"); break; case WAIT_UNTIL: b.append(" blocking until refresh"); break; case NONE: break; } return b.toString(); }
final ShardId shardId = entry.getKey(); final List<BulkItemRequest> requests = entry.getValue(); BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, bulkRequest.getRefreshPolicy(), requests.toArray(new BulkItemRequest[requests.size()])); bulkShardRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); bulkShardRequest.timeout(bulkRequest.timeout()); if (task != null) { bulkShardRequest.setParentTask(nodeId, task.getId());
@Override protected Tuple<BulkShardResponse, BulkShardRequest> shardOperationOnPrimary(MetaData metaData, BulkShardRequest request) { final IndexService indexService = indicesService.indexServiceSafe(request.index()); final IndexShard indexShard = indexService.shardSafe(request.shardId().id()); long[] preVersions = new long[request.items().length]; VersionType[] preVersionTypes = new VersionType[request.items().length]; Translog.Location location = null; for (int requestIndex = 0; requestIndex < request.items().length; requestIndex++) { BulkItemRequest item = request.items()[requestIndex]; item.request().copyContextAndHeadersFrom(request); if (item.request() instanceof IndexRequest) { applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); logger.trace("{} failed to execute bulk item (index) {}", e, request.shardId(), indexRequest); } else { logger.debug("{} failed to execute bulk item (index) {}", e, request.shardId(), indexRequest); } else { setResponse(item, new BulkItemResponse(item.id(), indexRequest.opType().lowercase(), new BulkItemResponse.Failure(request.index(), indexRequest.type(), indexRequest.id(), e))); applyVersion(request.items()[j], preVersions[j], preVersionTypes[j]); logger.trace("{} failed to execute bulk item (delete) {}", e, request.shardId(), deleteRequest); } else { logger.debug("{} failed to execute bulk item (delete) {}", e, request.shardId(), deleteRequest); } else {
long[] preVersions, VersionType[] preVersionTypes, Translog.Location location, int requestIndex) throws Exception { final DocWriteRequest itemRequest = request.items()[requestIndex].request(); preVersions[requestIndex] = itemRequest.version(); preVersionTypes[requestIndex] = itemRequest.versionType(); replicaRequest = request.items()[requestIndex]; break; case UPDATE: deleteRequest.version(deleteResult.getVersion()); assert deleteRequest.versionType().validateVersionForWrites(deleteRequest.version()); response = new DeleteResponse(request.shardId(), deleteRequest.type(), deleteRequest.id(), deleteResult.getVersion(), deleteResult.isFound()); replicaRequest = request.items()[requestIndex]; break; default: request.items()[requestIndex] = replicaRequest; if (operationResult == null) { // in case of noop update operation assert response.getResult() == DocWriteResponse.Result.NOOP if (isConflictException(failure)) { logger.trace((Supplier<?>) () -> new ParameterizedMessage("{} failed to execute bulk item ({}) {}", request.shardId(), docWriteRequest.opType().getLowercase(), request), failure); } else { logger.debug((Supplier<?>) () -> new ParameterizedMessage("{} failed to execute bulk item ({}) {}", request.shardId(), docWriteRequest.opType().getLowercase(), request), failure);
@Override protected void shardOperationOnReplica(BulkShardRequest request) { final ShardId shardId = request.shardId(); IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); IndexShard indexShard = indexService.shardSafe(shardId.id()); Translog.Location location = null; for (int i = 0; i < request.items().length; i++) { BulkItemRequest item = request.items()[i]; if (item == null || item.isIgnoreOnReplica()) { continue; processAfterWrite(request.refresh(), indexShard, location);
final ShardId shardId = entry.getKey(); final List<BulkItemRequest> requests = entry.getValue(); BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, bulkRequest.getRefreshPolicy(), requests.toArray(new BulkItemRequest[requests.size()])); bulkShardRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); bulkShardRequest.timeout(bulkRequest.timeout()); if (task != null) { bulkShardRequest.setParentTask(nodeId, task.getId());
final ShardId shardId = entry.getKey(); final List<BulkItemRequest> requests = entry.getValue(); BulkShardRequest bulkShardRequest = new BulkShardRequest(bulkRequest, shardId, bulkRequest.refresh(), requests.toArray(new BulkItemRequest[requests.size()])); bulkShardRequest.consistencyLevel(bulkRequest.consistencyLevel()); bulkShardRequest.timeout(bulkRequest.timeout()); shardBulkAction.execute(bulkShardRequest, new ActionListener<BulkShardResponse>() { @Override
@Override protected WriteReplicaResult<Request> shardOperationOnReplica( Request replicaRequest, IndexShard replica) throws Exception { BulkItemRequest[] itemRequests = new BulkItemRequest[1]; WriteRequest.RefreshPolicy refreshPolicy = replicaRequest.getRefreshPolicy(); itemRequests[0] = new BulkItemRequest(0, ((DocWriteRequest) replicaRequest)); BulkShardRequest bulkShardRequest = new BulkShardRequest(replicaRequest.shardId(), refreshPolicy, itemRequests); WriteReplicaResult<BulkShardRequest> result = shardBulkAction.shardOperationOnReplica(bulkShardRequest, replica); // a replica operation can never throw a document-level failure, // as the same document has been already indexed successfully in the primary return new WriteReplicaResult<>(replicaRequest, result.location, null, replica, logger); }
Engine.Result updateOperationResult = null; UpdateResponse updateResponse = null; BulkItemRequest replicaRequest = request.items()[requestIndex]; int maxAttempts = updateRequest.retryOnConflict(); for (int attemptCount = 0; attemptCount <= maxAttempts; attemptCount++) { IndexRequest indexRequest = translate.action(); MappingMetaData mappingMd = metaData.mappingOrDefault(indexRequest.type()); indexRequest.process(mappingMd, allowIdGeneration, request.index()); updateOperationResult = executeIndexRequestOnPrimary(indexRequest, primary, mappingUpdatedAction, this.clusterService, this.indicesService, metaData); if (updateOperationResult.hasFailure() == false) { Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(indexSourceAsBytes, true, updateIndexRequest.getContentType()); updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes)); replicaRequest = new BulkItemRequest(request.items()[requestIndex].id(), updateIndexRequest); break; case DELETE: deleteResponse.getVersion(), deleteResponse.getResult()); updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), deleteResponse.getVersion(), translate.updatedSourceAsMap(), translate.updateSourceContentType(), null)); replicaRequest = new BulkItemRequest(request.items()[requestIndex].id(), updateDeleteRequest); break; default: throw new IllegalStateException("Illegal update operation " +
new BulkItemResponse.Failure(request.index(), docWriteRequest.type(), docWriteRequest.id(), result.getFailure(), result.getSeqNo())); break;
public BulkShardRequest(ShardId shardId, RefreshPolicy refreshPolicy, BulkItemRequest[] items) { super(shardId); this.items = items; setRefreshPolicy(refreshPolicy); }
@Override protected WritePrimaryResult<BulkShardRequest, BulkShardResponse> shardOperationOnPrimary(BulkShardRequest request, IndexShard primary) throws Exception { ClusterStateObserver observer = new ClusterStateObserver(clusterService, request.timeout(), logger, threadPool.getThreadContext()); CheckedRunnable<Exception> waitForMappingUpdate = () -> { PlainActionFuture<Void> waitingFuture = new PlainActionFuture<>(); observer.waitForNextChange(new ClusterStateObserver.Listener() { @Override public void onNewClusterState(ClusterState state) { waitingFuture.onResponse(null); } @Override public void onClusterServiceClose() { waitingFuture.onFailure(new NodeClosedException(clusterService.localNode())); } @Override public void onTimeout(TimeValue timeout) { waitingFuture.onFailure( new MapperException("timed out while waiting for a dynamic mapping update")); } }); waitingFuture.get(); }; return performOnPrimary(request, primary, updateHelper, threadPool::absoluteTimeInMillis, new ConcreteMappingUpdatePerformer(), waitForMappingUpdate); }
final ShardId shardId = entry.getKey(); final List<BulkItemRequest> requests = entry.getValue(); BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, bulkRequest.getRefreshPolicy(), requests.toArray(new BulkItemRequest[requests.size()])); bulkShardRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); bulkShardRequest.timeout(bulkRequest.timeout()); if (task != null) { bulkShardRequest.setParentTask(nodeId, task.getId());
@Override protected WritePrimaryResult<Request, Response> shardOperationOnPrimary( Request request, final IndexShard primary) throws Exception { BulkItemRequest[] itemRequests = new BulkItemRequest[1]; WriteRequest.RefreshPolicy refreshPolicy = request.getRefreshPolicy(); request.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); itemRequests[0] = new BulkItemRequest(0, ((DocWriteRequest) request)); BulkShardRequest bulkShardRequest = new BulkShardRequest(request.shardId(), refreshPolicy, itemRequests); WritePrimaryResult<BulkShardRequest, BulkShardResponse> bulkResult = shardBulkAction.shardOperationOnPrimary(bulkShardRequest, primary); assert bulkResult.finalResponseIfSuccessful.getResponses().length == 1 : "expected only one bulk shard response"; BulkItemResponse itemResponse = bulkResult.finalResponseIfSuccessful.getResponses()[0]; final Response response; final Exception failure; if (itemResponse.isFailed()) { failure = itemResponse.getFailure().getCause(); response = null; } else { response = (Response) itemResponse.getResponse(); failure = null; } return new WritePrimaryResult<>(request, response, bulkResult.location, failure, primary, logger); }
private WriteResult<IndexResponse> shardIndexOperation(BulkShardRequest request, IndexRequest indexRequest, MetaData metaData, IndexShard indexShard, boolean processed) throws Throwable { // validate, if routing is required, that we got routing MappingMetaData mappingMd = metaData.index(request.index()).mappingOrDefault(indexRequest.type()); if (mappingMd != null && mappingMd.routing().required()) { if (indexRequest.routing() == null) { throw new RoutingMissingException(request.index(), indexRequest.type(), indexRequest.id()); } } if (!processed) { indexRequest.process(metaData, mappingMd, allowIdGeneration, request.index()); } return TransportIndexAction.executeIndexRequestOnPrimary(request, indexRequest, indexShard, mappingUpdatedAction); }
BulkShardRequest(ShardId shardId, RefreshPolicy refreshPolicy, BulkItemRequest[] items) { super(shardId); this.items = items; setRefreshPolicy(refreshPolicy); }
@Override protected WritePrimaryResult<BulkShardRequest, BulkShardResponse> shardOperationOnPrimary(BulkShardRequest request, IndexShard primary) throws Exception { ClusterStateObserver observer = new ClusterStateObserver(clusterService, request.timeout(), logger, threadPool.getThreadContext()); CheckedRunnable<Exception> waitForMappingUpdate = () -> { PlainActionFuture<Void> waitingFuture = new PlainActionFuture<>(); observer.waitForNextChange(new ClusterStateObserver.Listener() { @Override public void onNewClusterState(ClusterState state) { waitingFuture.onResponse(null); } @Override public void onClusterServiceClose() { waitingFuture.onFailure(new NodeClosedException(clusterService.localNode())); } @Override public void onTimeout(TimeValue timeout) { waitingFuture.onFailure( new MapperException("timed out while waiting for a dynamic mapping update")); } }); waitingFuture.get(); }; return performOnPrimary(request, primary, updateHelper, threadPool::absoluteTimeInMillis, new ConcreteMappingUpdatePerformer(), waitForMappingUpdate); }
final ShardId shardId = entry.getKey(); final List<BulkItemRequest> requests = entry.getValue(); BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, bulkRequest.getRefreshPolicy(), requests.toArray(new BulkItemRequest[requests.size()])); bulkShardRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); bulkShardRequest.timeout(bulkRequest.timeout()); if (task != null) { bulkShardRequest.setParentTask(nodeId, task.getId());