private void processFailBulkResponse(BulkResponse bulkResponse, boolean hasParent) { for (BulkItemResponse response : bulkResponse.getItems()) { if (!response.isFailed()) { continue; } if (response.getFailure().getStatus() == RestStatus.NOT_FOUND) { logger.warn(response.getFailureMessage()); } else { logger.error("全量导入数据有误 {}", response.getFailureMessage()); throw new RuntimeException("全量数据 etl 异常: " + response.getFailureMessage()); } } }
response.getType(), response.getIndex(), response.getFailureMessage() ); if(response.getFailure()!=null && response.getFailure().getStatus() == RestStatus.TOO_MANY_REQUESTS){ hasTooManyRequests =true;
for(BulkItemResponse response : bulkItemResponses.getItems()) { if(response.isFailed() && response.getFailure().getStatus() != RestStatus.NOT_FOUND) { log.error("Failed to execute ES query {}", response.getFailureMessage()); actualFailure = true;
private RestStatus getStatus(BulkByScrollResponse response) { /* * Return the highest numbered rest status under the assumption that higher numbered statuses are "more error" and thus more * interesting to the user. */ RestStatus status = RestStatus.OK; if (response.isTimedOut()) { status = RestStatus.REQUEST_TIMEOUT; } for (Failure failure : response.getBulkFailures()) { if (failure.getStatus().getStatus() > status.getStatus()) { status = failure.getStatus(); } } for (SearchFailure failure: response.getSearchFailures()) { RestStatus failureStatus = ExceptionsHelper.status(failure.getReason()); if (failureStatus.getStatus() > status.getStatus()) { status = failureStatus; } } return status; } }
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.startObject(opType.getLowercase()); if (failure == null) { response.innerToXContent(builder, params); builder.field(STATUS, response.status().getStatus()); } else { builder.field(_INDEX, failure.getIndex()); builder.field(_TYPE, failure.getType()); builder.field(_ID, failure.getId()); builder.field(STATUS, failure.getStatus().getStatus()); builder.startObject(ERROR); ElasticsearchException.generateThrowableXContent(builder, params, failure.getCause()); builder.endObject(); } builder.endObject(); builder.endObject(); return builder; }
@Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { boolean hasFatalError = false; //Do not consider version conficts to be errors. Ignore old versions if (response.hasFailures()) { for (BulkItemResponse itemResp : response.getItems()) { if (itemResp.isFailed()) { if (itemResp.getFailure().getStatus().equals(RestStatus.CONFLICT)) { LOGGER.info("Failed to index document in Elasticsearch: " + itemResp.getFailureMessage()); } else { hasFatalError = true; LOGGER.error("Failed to index document in Elasticsearch: " + itemResp.getFailureMessage()); } } } } if (hasFatalError) { sendFailed.set(true); } else { updateSuccessMetrics(response); } }
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.startObject(opType.getLowercase()); if (failure == null) { response.innerToXContent(builder, params); builder.field(STATUS, response.status().getStatus()); } else { builder.field(_INDEX, failure.getIndex()); builder.field(_TYPE, failure.getType()); builder.field(_ID, failure.getId()); builder.field(STATUS, failure.getStatus().getStatus()); builder.startObject(ERROR); ElasticsearchException.generateThrowableXContent(builder, params, failure.getCause()); builder.endObject(); } builder.endObject(); builder.endObject(); return builder; }
void logBulkFailure(BulkResponse response) { boolean isError = false; StringBuilder sb = new StringBuilder(); sb.append("Ignore indexing of some docs more recent versions has already been indexed"); for (BulkItemResponse item : response.getItems()) { if (item.isFailed()) { if (item.getFailure().getStatus() == RestStatus.CONFLICT) { sb.append("\n ").append(item.getFailureMessage()); } else { isError = true; } } } if (isError) { log.error(response.buildFailureMessage()); } else { log.debug(sb); } }
@Override public RestStatus status() { return failure == null ? response.status() : failure.getStatus(); }
private static boolean isRetryable(BulkItemResponse.Failure f) { return !fatalStatuses.contains(f.getStatus()); // todo Auth failures are also permanent. Need to see how they're surfaced, and decide how to handle. }
@Override public RestStatus status() { return failure == null ? response.status() : failure.getStatus(); }
@Override public RestStatus status() { return failure == null ? response.status() : failure.getStatus(); }
private void recordFailure(Failure failure, List<Failure> failures) { if (failure.getStatus() == CONFLICT) { worker.countVersionConflict(); if (false == mainRequest.isAbortOnVersionConflict()) { return; } } failures.add(failure); }
private void recordFailure(Failure failure, List<Failure> failures) { if (failure.getStatus() == CONFLICT) { task.countVersionConflict(); if (false == mainRequest.isAbortOnVersionConflict()) { return; } } failures.add(failure); }
private BulkResponse getRespWithFailedDocument(RestStatus status) { BulkResponse response = mock(BulkResponse.class); when(response.hasFailures()).thenReturn(true); BulkItemResponse itemResp = mock(BulkItemResponse.class); when(itemResp.isFailed()).thenReturn(true); BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class); when(failure.getStatus()).thenReturn(status); when(itemResp.getFailure()).thenReturn(failure); BulkItemResponse[] itemResponses = new BulkItemResponse[]{itemResp}; when(response.getItems()).thenReturn(itemResponses); return response; } }
private void recordFailure(Failure failure, List<Failure> failures) { if (failure.getStatus() == CONFLICT) { task.countVersionConflict(); if (false == mainRequest.isAbortOnVersionConflict()) { return; } } failures.add(failure); }
void add(Event e, BulkItemResponse.Failure failure) { if (!logger.isInfoEnabled()) { return; } final String message = "status=" + failure.getStatus() + " message=" + failure.getMessage(); errorMessageToEvents.put(message, RedactableArgument.user(e).toString()); }
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.startObject(opType.getLowercase()); if (failure == null) { response.innerToXContent(builder, params); builder.field(STATUS, response.status().getStatus()); } else { builder.field(_INDEX, failure.getIndex()); builder.field(_TYPE, failure.getType()); builder.field(_ID, failure.getId()); builder.field(STATUS, failure.getStatus().getStatus()); builder.startObject(ERROR); ElasticsearchException.generateThrowableXContent(builder, params, failure.getCause()); builder.endObject(); } builder.endObject(); builder.endObject(); return builder; }
@Override public RestStatus status() { return failure == null ? response.status() : failure.getStatus(); }
/** * 批量提交 * * @param bulkRequestBuilder * @return */ private static boolean commitBulkRequest(BulkRequestBuilder bulkRequestBuilder) { if (bulkRequestBuilder.numberOfActions() > 0) { BulkResponse response = bulkRequestBuilder.execute().actionGet(); if (response.hasFailures()) { for (BulkItemResponse itemResponse : response.getItems()) { if (!itemResponse.isFailed()) { continue; } if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { logger.warn(itemResponse.getFailureMessage()); } else { logger.error("ES sync commit error: {}", itemResponse.getFailureMessage()); } } } return !response.hasFailures(); } return true; }