private void processFailBulkResponse(BulkResponse bulkResponse, boolean hasParent) { for (BulkItemResponse response : bulkResponse.getItems()) { if (!response.isFailed()) { continue; } if (response.getFailure().getStatus() == RestStatus.NOT_FOUND) { logger.warn(response.getFailureMessage()); } else { logger.error("全量导入数据有误 {}", response.getFailureMessage()); throw new RuntimeException("全量数据 etl 异常: " + response.getFailureMessage()); } } }
private void checkForBulkUpdateFailure(BulkResponse bulkResponse) { if (bulkResponse.hasFailures()) { Map<String, String> failedDocuments = new HashMap<>(); for (BulkItemResponse item : bulkResponse.getItems()) { if (item.isFailed()) failedDocuments.put(item.getId(), item.getFailureMessage()); } throw new ElasticsearchException( "Bulk indexing has failures. Use ElasticsearchException.getFailedDocuments() for detailed messages [" + failedDocuments + "]", failedDocuments); } }
@Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { if (response.hasFailures()) { logger.error("Bulk processor failed. {}", response.buildFailureMessage()); MongoDBRiverHelper.setRiverStatus(client, definition.getRiverName(), Status.IMPORT_FAILED); request.requests().clear(); bulkProcessor.close(); river.close(); } else { documentCount.addAndGet(response.getItems().length); logStatistics(response.getTookInMillis()); deletedDocuments.set(0); updatedDocuments.set(0); insertedDocuments.set(0); if (logger.isTraceEnabled()) { logger.trace("afterBulk - bulk [{}] success [{} items] [{} ms] total [{}]", executionId, response.getItems().length, response.getTookInMillis(), documentCount.get()); } } } };
/** * 批量提交 * * @param bulkRequestBuilder * @return */ private static boolean commitBulkRequest(BulkRequestBuilder bulkRequestBuilder) { if (bulkRequestBuilder.numberOfActions() > 0) { BulkResponse response = bulkRequestBuilder.execute().actionGet(); if (response.hasFailures()) { for (BulkItemResponse itemResponse : response.getItems()) { if (!itemResponse.isFailed()) { continue; } if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) { logger.warn(itemResponse.getFailureMessage()); } else { logger.error("ES sync commit error: {}", itemResponse.getFailureMessage()); } } } return !response.hasFailures(); } return true; }
@Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { stopProfiler(request); List<DocId> successDocIds = new ArrayList<>(); for (BulkItemResponse item : response.getItems()) { if (item.isFailed()) { LOGGER.error("index [{}], type [{}], id [{}], message [{}]", item.getIndex(), item.getType(), item.getId(), item.getFailureMessage()); } else { result.incrementSuccess(); successDocIds.add(new DocId(item.getIndex(), item.getType(), item.getId())); } } indexingListener.onSuccess(successDocIds); }
if (bulkItemResponses.hasFailures()) { boolean actualFailure = false; for(BulkItemResponse response : bulkItemResponses.getItems()) {
for (BulkItemResponse r: bulkResponse.getItems()) { String id = r.getId(); ActionWriteResponse response = r.getResponse();
if (response.hasFailures()) { BulkItemResponse[] responses = response.getItems(); if (responses != null && responses.length > 0) { for (int i = responses.length - 1; i >= 0; i--) {
if (response.hasFailures()) { BulkItemResponse[] responses = response.getItems(); if (responses != null && responses.length > 0) { for (int i = responses.length - 1; i >= 0; i--) {
@Override public void onResponse(BulkResponse response) { BulkItemResponse[] items = response.getItems(); for (int i = 0; i < items.length; i++) { itemResponses.add(originalSlots[i], response.getItems()[i]); } actionListener.onResponse(new BulkResponse( itemResponses.toArray(new BulkItemResponse[itemResponses.size()]), response.getTook().getMillis(), ingestTookInMillis)); }
private BulkRequest createBulkRequestForRetry(BulkResponse bulkItemResponses) { BulkRequest requestToReissue = new BulkRequest(); int index = 0; for (BulkItemResponse bulkItemResponse : bulkItemResponses.getItems()) { if (bulkItemResponse.isFailed()) { requestToReissue.add(currentBulkRequest.requests().get(index)); } index++; } return requestToReissue; }
public static <Response extends ReplicationResponse & WriteResponse> ActionListener<BulkResponse> wrapBulkResponse(ActionListener<Response> listener) { return ActionListener.wrap(bulkItemResponses -> { assert bulkItemResponses.getItems().length == 1 : "expected only one item in bulk request"; BulkItemResponse bulkItemResponse = bulkItemResponses.getItems()[0]; if (bulkItemResponse.isFailed() == false) { final DocWriteResponse response = bulkItemResponse.getResponse(); listener.onResponse((Response) response); } else { listener.onFailure(bulkItemResponse.getFailure().getCause()); } }, listener::onFailure); }
ActionListener<BulkResponse> wrapActionListenerIfNeeded(long ingestTookInMillis, ActionListener<BulkResponse> actionListener) { if (itemResponses.isEmpty()) { return ActionListener.wrap( response -> actionListener.onResponse(new BulkResponse(response.getItems(), response.getTook().getMillis(), ingestTookInMillis)), actionListener::onFailure); } else { return new IngestBulkResponseListener(ingestTookInMillis, originalSlots, itemResponses, actionListener); } }
for (BulkItemResponse r: bulkResponse.getItems()) { String id = r.getId(); DocWriteResponse response = r.getResponse();
return response != null ? response.getItems().length : 0;
@Override public void onResponse(BulkResponse response) { BulkItemResponse[] items = response.getItems(); for (int i = 0; i < items.length; i++) { itemResponses.add(originalSlots[i], response.getItems()[i]); } actionListener.onResponse(new BulkResponse( itemResponses.toArray(new BulkItemResponse[itemResponses.size()]), response.getTook().getMillis(), ingestTookInMillis)); }
private BulkRequest createBulkRequestForRetry(BulkResponse bulkItemResponses) { BulkRequest requestToReissue = new BulkRequest(); int index = 0; for (BulkItemResponse bulkItemResponse : bulkItemResponses.getItems()) { if (bulkItemResponse.isFailed()) { requestToReissue.add(currentBulkRequest.requests().get(index)); } index++; } return requestToReissue; }
private ActionListener<BulkResponse> wrapBulkResponse(ActionListener<Response> listener) { return ActionListener.wrap(bulkItemResponses -> { assert bulkItemResponses.getItems().length == 1 : "expected only one item in bulk request"; BulkItemResponse bulkItemResponse = bulkItemResponses.getItems()[0]; if (bulkItemResponse.isFailed() == false) { final DocWriteResponse response = bulkItemResponse.getResponse(); listener.onResponse((Response) response); } else { listener.onFailure(bulkItemResponse.getFailure().getCause()); } }, listener::onFailure); }
ActionListener<BulkResponse> wrapActionListenerIfNeeded(long ingestTookInMillis, ActionListener<BulkResponse> actionListener) { if (itemResponses.isEmpty()) { return ActionListener.wrap( response -> actionListener.onResponse( new BulkResponse(response.getItems(), response.getTookInMillis(), ingestTookInMillis)), actionListener::onFailure); } else { return new IngestBulkResponseListener(ingestTookInMillis, originalSlots, itemResponses, actionListener); } }
private BulkResponse getRespWithFailedDocument(RestStatus status) { BulkResponse response = mock(BulkResponse.class); when(response.hasFailures()).thenReturn(true); BulkItemResponse itemResp = mock(BulkItemResponse.class); when(itemResp.isFailed()).thenReturn(true); BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class); when(failure.getStatus()).thenReturn(status); when(itemResp.getFailure()).thenReturn(failure); BulkItemResponse[] itemResponses = new BulkItemResponse[]{itemResp}; when(response.getItems()).thenReturn(itemResponses); return response; } }