@Override public DeleteOperationResponse deleteById(String index, String type, List<String> ids) throws IOException { BulkRequest bulk = new BulkRequest(); for (int idx = 0; idx < ids.size(); idx++) { DeleteRequest request = new DeleteRequest(index, type, ids.get(idx)); bulk.add(request); } BulkResponse response = highLevelClient.bulk(bulk); DeleteOperationResponse dor = new DeleteOperationResponse(response.getTookInMillis()); return dor; }
new RetryUtil<BulkResponse>().retryOnException(() -> { try { return elasticSearchClient.bulk(bulkRequest); } catch (IOException e) { throw new RuntimeException(e);
@Override public void bulkUpdate(List<UpdateQuery> queries) { BulkRequest bulkRequest = new BulkRequest(); for (UpdateQuery query : queries) { bulkRequest.add(prepareUpdate(query)); } try { checkForBulkUpdateFailure(client.bulk(bulkRequest)); } catch (IOException e) { throw new ElasticsearchException("Error while bulk for request: " + bulkRequest.toString(), e); } }
@Override public void bulkIndex(List<IndexQuery> queries) { BulkRequest bulkRequest = new BulkRequest(); for (IndexQuery query : queries) { bulkRequest.add(prepareIndex(query)); } try { checkForBulkUpdateFailure(client.bulk(bulkRequest)); } catch (IOException e) { throw new ElasticsearchException("Error while bulk for request: " + bulkRequest.toString(), e); } }
@Override public IndexOperationResponse add(List<IndexOperationRequest> operations) throws IOException { BulkRequest bulkRequest = new BulkRequest(); for (int index = 0; index < operations.size(); index++) { IndexOperationRequest or = operations.get(index); IndexRequest indexRequest = new IndexRequest(or.getIndex(), or.getType(), or.getId()) .source(or.getFields()); bulkRequest.add(indexRequest); } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse response = highLevelClient.bulk(bulkRequest); IndexOperationResponse retVal = new IndexOperationResponse(response.getTookInMillis(), response.getIngestTookInMillis()); return retVal; }
BulkResponse response; try { response = client.bulk(request); checkForBulkUpdateFailure(response); } catch (IOException e) {
public void bulkDocument(String type, Map<String, Map<String, Object>> sources) { try { if (this.instance() == null) return; BulkRequest requests = new BulkRequest(); Iterator<String> it = sources.keySet().iterator(); int count = 0; while(it.hasNext()) { count++; String next = it.next(); IndexRequest request = new IndexRequest(name, type, next); request.source(sources.get(next)); requests.add(request); if (count % 1000 == 0) { client.bulk(requests, RequestOptions.DEFAULT); requests.requests().clear(); count = 0; } } if (requests.numberOfActions() > 0) client.bulk(requests, RequestOptions.DEFAULT); } catch (IOException e) { log.error(e.getMessage()); } }
public void bulkDeleteDocument(String type, List<Integer> ids) { try { if (this.instance() == null) return; BulkRequest requests = new BulkRequest(); int count = 0; for (Integer id: ids) { count++; DeleteRequest request = new DeleteRequest(name, type, String.valueOf(id)); requests.add(request); if (count % 1000 == 0) { client.bulk(requests, RequestOptions.DEFAULT); requests.requests().clear(); count = 0; } } if (requests.numberOfActions() > 0) client.bulk(requests, RequestOptions.DEFAULT); } catch (IOException e) { log.error(e.getMessage()); } }
private void setupElasticsearchToSucceed() throws IOException { final String documentId = UUID.randomUUID().toString(); final boolean isFailed = false; final int itemID = 0; // the write response will contain what is used as the document ID DocWriteResponse writeResponse = mock(DocWriteResponse.class); when(writeResponse.getId()).thenReturn(documentId); // define the item level response BulkItemResponse itemResponse = mock(BulkItemResponse.class); when(itemResponse.isFailed()).thenReturn(isFailed); when(itemResponse.getItemId()).thenReturn(itemID); when(itemResponse.getResponse()).thenReturn(writeResponse); List<BulkItemResponse> itemsResponses = Collections.singletonList(itemResponse); // define the bulk response to indicate success BulkResponse response = mock(BulkResponse.class); when(response.iterator()).thenReturn(itemsResponses.iterator()); when(response.hasFailures()).thenReturn(isFailed); // have the client return the mock response when(highLevelClient.bulk(any(BulkRequest.class))).thenReturn(response); }
@Override public BulkDocumentWriterResults<D> write() { BulkDocumentWriterResults<D> results = new BulkDocumentWriterResults<>(); try { // create an index request for each document BulkRequest bulkRequest = new BulkRequest(); bulkRequest.setRefreshPolicy(refreshPolicy); for(Indexable doc: documents) { DocWriteRequest request = createRequest(doc.document, doc.index); bulkRequest.add(request); } // submit the request and handle the response BulkResponse bulkResponse = client.getHighLevelClient().bulk(bulkRequest); handleBulkResponse(bulkResponse, documents, results); } catch(IOException e) { // assume all documents have failed for(Indexable indexable: documents) { D failed = indexable.document; results.addFailure(failed, e, ExceptionUtils.getRootCauseMessage(e)); } LOG.error("Failed to submit bulk request; all documents failed", e); } finally { // flush all documents no matter which ones succeeded or failed documents.clear(); } LOG.debug("Wrote document(s) to Elasticsearch; batchSize={}, success={}, failed={}", documents.size(), results.getSuccesses().size(), results.getFailures().size()); return results; }
private void setupElasticsearchToFail() throws IOException { final String errorMessage = "error message"; final Exception cause = new Exception("test exception"); final boolean isFailed = true; final int itemID = 0; // define the item failure BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class); when(failure.getCause()).thenReturn(cause); when(failure.getMessage()).thenReturn(errorMessage); // define the item level response BulkItemResponse itemResponse = mock(BulkItemResponse.class); when(itemResponse.isFailed()).thenReturn(isFailed); when(itemResponse.getItemId()).thenReturn(itemID); when(itemResponse.getFailure()).thenReturn(failure); when(itemResponse.getFailureMessage()).thenReturn("error message"); List<BulkItemResponse> itemsResponses = Collections.singletonList(itemResponse); // define the bulk response to indicate failure BulkResponse response = mock(BulkResponse.class); when(response.iterator()).thenReturn(itemsResponses.iterator()); when(response.hasFailures()).thenReturn(isFailed); // have the client return the mock response when(highLevelClient.bulk(any(BulkRequest.class))).thenReturn(response); }
@Override public BulkResponse bulk(BulkRequest request) { try { return client.bulk(request, RequestOptions.DEFAULT); } catch (IOException e) { throw new NuxeoException(e); } }
@Override public DeleteOperationResponse deleteById(String index, String type, List<String> ids) throws IOException { BulkRequest bulk = new BulkRequest(); for (int idx = 0; idx < ids.size(); idx++) { DeleteRequest request = new DeleteRequest(index, type, ids.get(idx)); bulk.add(request); } BulkResponse response = highLevelClient.bulk(bulk); DeleteOperationResponse dor = new DeleteOperationResponse(response.getTookInMillis()); return dor; }
@Override public BulkResponse deleteIndex(List<IndexBuilder> batch) throws Exception { BulkRequest request = new BulkRequest(); Optional.ofNullable(batch).orElseThrow(NullPointerException::new).forEach(x ->{ request.add(new DeleteRequest(x.getIndex(), x.getType(), x.getId())); }); log.debug(request.getDescription()); return client().bulk(request); }
@Override public BulkResponse createIndex(List<IndexBuilder> batch) throws Exception { BulkRequest request = new BulkRequest(); Optional.ofNullable(batch).orElseThrow(NullPointerException::new).forEach(x ->{ request.add(new IndexRequest(x.getIndex(), x.getType(), x.getId()).source(JSON.toJSONString(x.getSource()), XContentType.JSON)); }); log.debug(request.getDescription()); return client().bulk(request); }
@Override public BulkResponse updateIndex(List<IndexBuilder> batch) throws Exception { BulkRequest request = new BulkRequest(); Optional.ofNullable(batch).orElseThrow(NullPointerException::new).forEach(x ->{ request.add(new UpdateRequest(x.getIndex(), x.getType(), x.getId()).doc(JSON.toJSONString(x.getSource()), XContentType.JSON)); }); log.debug(request.getDescription()); return client().bulk(request); }
@Override public BulkResponse upsertIndex(List<IndexBuilder> batch) throws Exception { BulkRequest request = new BulkRequest(); Optional.ofNullable(batch).orElseThrow(NullPointerException::new).forEach(x ->{ request.add(new UpdateRequest(x.getIndex(), x.getType(), x.getId()).docAsUpsert(true).doc(JSON.toJSONString(x.getSource()), XContentType.JSON)); }); log.debug(request.getDescription()); return client().bulk(request); }
/** @return true if all bulk records where flushed successfully */ public boolean flush() { boolean success = true; if (bulkRequest.numberOfActions() > 0) { try { final BulkResponse responses = client.bulk(bulkRequest, RequestOptions.DEFAULT); success = checkBulkResponses(responses); } catch (IOException e) { throw new ElasticsearchExporterException("Failed to flush bulk", e); } if (success) { // all records where flushed, create new bulk request, otherwise retry next time bulkRequest = new BulkRequest(); } } return success; }
@Override public IndexOperationResponse add(List<IndexOperationRequest> operations) throws IOException { BulkRequest bulkRequest = new BulkRequest(); for (int index = 0; index < operations.size(); index++) { IndexOperationRequest or = operations.get(index); IndexRequest indexRequest = new IndexRequest(or.getIndex(), or.getType(), or.getId()) .source(or.getFields()); bulkRequest.add(indexRequest); } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse response = highLevelClient.bulk(bulkRequest); IndexOperationResponse retVal = new IndexOperationResponse(response.getTookInMillis(), response.getIngestTookInMillis()); return retVal; }
public void bulk() throws IOException { BulkRequest request = new BulkRequest(); request.add(new IndexRequest("posts", "doc", "1") .source(XContentType.JSON,"field", "foo")); request.add(new IndexRequest("posts", "doc", "2") .source(XContentType.JSON,"field", "bar")); request.add(new IndexRequest("posts", "doc", "3") .source(XContentType.JSON,"field", "baz")); BulkResponse bulkResponse = client.bulk(request); } }