Refine search
private void processFailBulkResponse(BulkResponse bulkResponse, boolean hasParent) { for (BulkItemResponse response : bulkResponse.getItems()) { if (!response.isFailed()) { continue; } if (response.getFailure().getStatus() == RestStatus.NOT_FOUND) { logger.warn(response.getFailureMessage()); } else { logger.error("全量导入数据有误 {}", response.getFailureMessage()); throw new RuntimeException("全量数据 etl 异常: " + response.getFailureMessage()); } } }
@Nullable @Override public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) { if (bulkItemResponse.isFailed()) { return new Exception(bulkItemResponse.getFailure().getMessage()); } else { return null; } }
@Override public Object answer(InvocationOnMock invocationOnMock) throws Throwable { while (nextBulkRequest.numberOfActions() > 0) { // wait until we are allowed to continue with the flushing flushLatch.await(); // create a copy of the accumulated mock requests, so that // re-added requests from the failure handler are included in the next bulk BulkRequest currentBulkRequest = nextBulkRequest; nextBulkRequest = new BulkRequest(); listener.beforeBulk(123L, currentBulkRequest); if (nextBulkFailure == null) { BulkItemResponse[] mockResponses = new BulkItemResponse[currentBulkRequest.requests().size()]; for (int i = 0; i < currentBulkRequest.requests().size(); i++) { Throwable mockItemFailure = mockItemFailuresList.get(i); if (mockItemFailure == null) { // the mock response for the item is success mockResponses[i] = new BulkItemResponse(i, "opType", mock(ActionResponse.class)); } else { // the mock response for the item is failure mockResponses[i] = new BulkItemResponse(i, "opType", new BulkItemResponse.Failure("index", "type", "id", mockItemFailure)); } } listener.afterBulk(123L, currentBulkRequest, new BulkResponse(mockResponses, 1000L)); } else { listener.afterBulk(123L, currentBulkRequest, nextBulkFailure); } } return null; } }).when(mockBulkProcessor).flush();
for ( BulkItemResponse response : responses ) { if ( response.isFailed() ) { logger.error( "Unable to index id={}, type={}, index={}, failureMessage={} ", response.getId(), response.getType(), response.getIndex(), response.getFailureMessage() ); if(response.getFailure()!=null && response.getFailure().getStatus() == RestStatus.TOO_MANY_REQUESTS){ hasTooManyRequests =true;
public static <Response extends ReplicationResponse & WriteResponse> ActionListener<BulkResponse> wrapBulkResponse(ActionListener<Response> listener) { return ActionListener.wrap(bulkItemResponses -> { assert bulkItemResponses.getItems().length == 1 : "expected only one item in bulk request"; BulkItemResponse bulkItemResponse = bulkItemResponses.getItems()[0]; if (bulkItemResponse.isFailed() == false) { final DocWriteResponse response = bulkItemResponse.getResponse(); listener.onResponse((Response) response); } else { listener.onFailure(bulkItemResponse.getFailure().getCause()); } }, listener::onFailure); }
private void setupElasticsearchToFail() throws IOException { final String errorMessage = "error message"; final Exception cause = new Exception("test exception"); final boolean isFailed = true; final int itemID = 0; // define the item failure BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class); when(failure.getCause()).thenReturn(cause); when(failure.getMessage()).thenReturn(errorMessage); // define the item level response BulkItemResponse itemResponse = mock(BulkItemResponse.class); when(itemResponse.isFailed()).thenReturn(isFailed); when(itemResponse.getItemId()).thenReturn(itemID); when(itemResponse.getFailure()).thenReturn(failure); when(itemResponse.getFailureMessage()).thenReturn("error message"); List<BulkItemResponse> itemsResponses = Collections.singletonList(itemResponse); // define the bulk response to indicate failure BulkResponse response = mock(BulkResponse.class); when(response.iterator()).thenReturn(itemsResponses.iterator()); when(response.hasFailures()).thenReturn(isFailed); // have the client return the mock response when(highLevelClient.bulk(any(BulkRequest.class))).thenReturn(response); }
for (BulkItemResponse itemResponse : shard.getValue()) { builder.startObject(); builder.startObject(itemResponse.getOpType()); builder.field(Fields._INDEX, itemResponse.getIndex()); builder.field(Fields._TYPE, itemResponse.getType()); builder.field(Fields._ID, itemResponse.getId()); long version = itemResponse.getVersion(); if (itemResponse.isFailed()) { builder.field(Fields.ERROR, itemResponse.getFailure().getMessage()); } else { builder.field(Fields.OK, true);
if (!bulkItemResponse.isFailed()) { boolean add = ids.add(bulkItemResponse.getId()); assert add : "ID: " + bulkItemResponse.getId() + " already used"; } else { throw new ElasticsearchException("bulk request failure, id: [" + bulkItemResponse.getFailure().getId() + "] message: " + bulkItemResponse.getFailure().getMessage());
for (int i = responses.length - 1; i >= 0; i--) { final FlowFile flowFile = flowFilesToTransfer.get(i); if (responses[i].isFailed()) { logger.error("Failed to insert {} into Elasticsearch due to {}, transferring to failure", new Object[]{flowFile, responses[i].getFailure().getMessage()}); session.transfer(flowFile, REL_FAILURE); session.getProvenanceReporter().send(flowFile, context.getProperty(HOSTS).evaluateAttributeExpressions().getValue() + "/" + responses[i].getIndex()); session.transfer(flowFile, REL_SUCCESS);
for (int i = responses.length - 1; i >= 0; i--) { final BulkItemResponse item = responses[i]; final FlowFile flowFile = flowFilesToTransfer.get(item.getItemId()); if (item.isFailed()) { logger.warn("Failed to insert {} into Elasticsearch due to {}, transferring to failure", new Object[]{flowFile, item.getFailure().getMessage()}); session.transfer(flowFile, REL_FAILURE);
/** * Abort this request, and store a {@link org.elasticsearch.action.bulk.BulkItemResponse.Failure} response. * * @param index The concrete index that was resolved for this request * @param cause The cause of the rejection (may not be null) * @throws IllegalStateException If a response already exists for this request */ public void abort(String index, Exception cause) { if (primaryResponse == null) { final BulkItemResponse.Failure failure = new BulkItemResponse.Failure(index, request.type(), request.id(), Objects.requireNonNull(cause), true); setPrimaryResponse(new BulkItemResponse(id, request.opType(), failure)); } else { assert primaryResponse.isFailed() && primaryResponse.getFailure().isAborted() : "response [" + Strings.toString(primaryResponse) + "]; cause [" + cause + "]"; if (primaryResponse.isFailed() && primaryResponse.getFailure().isAborted()) { primaryResponse.getFailure().getCause().addSuppressed(cause); } else { throw new IllegalStateException( "aborting item that with response [" + primaryResponse + "] that was previously processed", cause); } } }
public static Translog.Location performOnReplica(BulkShardRequest request, IndexShard replica) throws Exception { Translog.Location location = null; for (int i = 0; i < request.items().length; i++) { BulkItemRequest item = request.items()[i]; final Engine.Result operationResult; DocWriteRequest docWriteRequest = item.request(); switch (replicaItemExecutionMode(item, i)) { case NORMAL: final DocWriteResponse primaryResponse = item.getPrimaryResponse().getResponse(); operationResult = performOpOnReplica(primaryResponse, docWriteRequest, replica); assert operationResult != null : "operation result must never be null when primary response has no failure"; location = syncOperationResultOrThrow(operationResult, location); break; case NOOP: break; case FAILURE: final BulkItemResponse.Failure failure = item.getPrimaryResponse().getFailure(); assert failure.getSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "seq no must be assigned"; operationResult = replica.markSeqNoAsNoop(failure.getSeqNo(), failure.getMessage()); assert operationResult != null : "operation result must never be null when primary response has no failure"; location = syncOperationResultOrThrow(operationResult, location); break; default: throw new IllegalStateException("illegal replica item execution mode for: " + docWriteRequest); } } return location; }
/** * Checks if is index avaialble. * * @param bulkItemResponses the bulk item responses * @return the boolean */ private Boolean isIndexAvaialble(BulkItemResponse[] bulkItemResponses) { // System.out.println(bulkItemResponses[0].getFailureMessage()); // System.out.println(bulkItemResponses[0].getFailure().getMessage()); return null == Arrays.stream(bulkItemResponses) .filter(x -> x.getFailure().getCause().getMessage().contains("no such index")).findAny().orElse(null); }
throw new AssertionError("unknown result type :" + result.getResultType()); executionResult = new BulkItemResponse(current.id(), current.request().opType(), response); executionResult.getResponse().setShardInfo(new ReplicationResponse.ShardInfo()); locationToSync = TransportWriteAction.locationToSync(locationToSync, result.getTranslogLocation()); break; case FAILURE: executionResult = new BulkItemResponse(current.id(), docWriteRequest.opType(), new BulkItemResponse.Failure(request.index(), docWriteRequest.type(), docWriteRequest.id(), result.getFailure(), result.getSeqNo())); break;
/** * The index name of the action. */ public String getIndex() { if (failure != null) { return failure.getIndex(); } return response.getIndex(); }
/** * The id of the action. */ public String getId() { if (failure != null) { return failure.getId(); } return response.getId(); }
@Override public void writeTo(StreamOutput out) throws IOException { out.writeString(getIndex()); out.writeString(getType()); out.writeOptionalString(getId()); out.writeException(getCause()); if (out.getVersion().onOrAfter(Version.V_6_0_0_alpha1)) { out.writeZLong(getSeqNo()); } if (supportsAbortedFlag(out.getVersion())) { out.writeBoolean(aborted); } }
/** * The failure message, {@code null} if it did not fail. */ public String getFailureMessage() { if (failure != null) { return failure.getMessage(); } return null; }
@Override public RestStatus status() { return failure == null ? response.status() : failure.getStatus(); }
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(TOOK_FIELD, took.millis()); builder.field(TIMED_OUT_FIELD, timedOut); status.innerXContent(builder, params); builder.startArray("failures"); for (Failure failure: bulkFailures) { builder.startObject(); failure.toXContent(builder, params); builder.endObject(); } for (ScrollableHitSource.SearchFailure failure: searchFailures) { failure.toXContent(builder, params); } builder.endArray(); return builder; }