public SearchScrollRequestBuilder getScrollBuilder(String scrollId){ return esProvider.getClient().prepareSearchScroll( scrollId ); }
private List<SearchHit> scrollTillLimit(TableInJoinRequestBuilder tableInJoinRequest, Integer hintLimit) { SearchResponse scrollResp = scrollOneTimeWithMax(client,tableInJoinRequest); updateMetaSearchResults(scrollResp); List<SearchHit> hitsWithScan = new ArrayList<>(); int curentNumOfResults = 0; SearchHit[] hits = scrollResp.getHits().getHits(); if (hintLimit == null) hintLimit = MAX_RESULTS_FOR_FIRST_TABLE; while (hits.length != 0 && curentNumOfResults < hintLimit) { curentNumOfResults += hits.length; Collections.addAll(hitsWithScan, hits); if (curentNumOfResults >= MAX_RESULTS_FOR_FIRST_TABLE) { //todo: log or exception? System.out.println("too many results for first table, stoping at:" + curentNumOfResults); break; } scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); hits = scrollResp.getHits().getHits(); } return hitsWithScan; }
break; scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); hits = scrollResp.getHits().getHits(); break; scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); hits = scrollResp.getHits().getHits();
/** * Delete documents using a query. Check what would be deleted first with a normal search query! * Elasticsearch once provided a native prepareDeleteByQuery method, but this was removed * in later versions. Instead, there is a plugin which iterates over search results, * see https://www.elastic.co/guide/en/elasticsearch/plugins/current/plugins-delete-by-query.html * We simulate the same behaviour here without the need of that plugin. * * @param q * @return delete document count */ public int deleteByQuery(String indexName, final QueryBuilder q) { Map<String, String> ids = new TreeMap<>(); // FIXME: deprecated, "will be removed in 3.0, you should do a regular scroll instead, ordered by `_doc`" @SuppressWarnings("deprecation") SearchResponse response = elasticsearchClient.prepareSearch(indexName).setSearchType(SearchType.SCAN) .setScroll(new TimeValue(60000)).setQuery(q).setSize(100).execute().actionGet(); while (true) { // accumulate the ids here, don't delete them right now to prevent an interference of the delete with the // scroll for (SearchHit hit : response.getHits().getHits()) { ids.put(hit.getId(), hit.getType()); } response = elasticsearchClient.prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000)) .execute().actionGet(); // termination if (response.getHits().getHits().length == 0) break; } return deleteBulk(indexName, ids); }
break; responseForSecondTable = client.prepareSearchScroll(responseForSecondTable.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); secondQueryHits = responseForSecondTable.getHits().getHits(); scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); hits = scrollResp.getHits().getHits();
searchResponse = client.prepareSearchScroll(searchResponse.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); } else break; } else {
firstTableResponse = client.prepareSearchScroll(firstTableResponse.getScrollId()).setScroll(new TimeValue(600000)).get(); else finishedWithFirstTable = true;
/** * Get all the indexed documents (no paginated results). Results are not sorted. */ public List<SearchHit> getDocuments(IndexType indexType) { SearchRequestBuilder req = SHARED_NODE.client().prepareSearch(indexType.getIndex()).setTypes(indexType.getType()).setQuery(matchAllQuery()); EsUtils.optimizeScrollRequest(req); req.setScroll(new TimeValue(60000)) .setSize(100); SearchResponse response = req.get(); List<SearchHit> result = newArrayList(); while (true) { Iterables.addAll(result, response.getHits()); response = SHARED_NODE.client().prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); // Break condition: No hits are returned if (response.getHits().getHits().length == 0) { break; } } return result; }
ids.put(hit.getId(), hit.getType()); response = elasticsearchClient.prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000)) .execute().actionGet();
@Override protected SearchResponse scrollSearchResponse(final String scrollId) { return _client.prepareSearchScroll(scrollId).setScroll(ElasticSearchDataContext.TIMEOUT_SCROLL) .execute().actionGet(); } }
@Override protected SearchResponse scrollSearchResponse(final String scrollId) { return _client.prepareSearchScroll(scrollId).setScroll(ElasticSearchDataContext.TIMEOUT_SCROLL) .execute().actionGet(); } }
@Override protected SearchResponse getNextSearchResponse(String scrollId) { try { return client.prepareSearchScroll(scrollId) .setScroll(scrollKeepAlive) .execute().actionGet(); } catch (Exception ex) { throw new VertexiumException("Failed to request more items from scroll " + scrollId, ex); } }
@Override public SearchResponse next() { SearchResponse result = searchResponseFuture.actionGet(); searchResponseFuture = client.prepareSearchScroll(result.getScrollId()) .setScroll(TIMEOUT) .execute(); return result; }
@Override protected SearchResponse getNextSearchResponse(String scrollId) { try { return client.prepareSearchScroll(scrollId) .setScroll(scrollKeepAlive) .execute().actionGet(); } catch (Exception ex) { throw new VertexiumException("Failed to request more items from scroll " + scrollId, ex); } }
@Override protected SearchResponse getNextSearchResponse(String scrollId) { try { return client.prepareSearchScroll(scrollId) .setScroll(scrollKeepAlive) .execute().actionGet(); } catch (Exception ex) { throw new VertexiumException("Failed to request more items from scroll " + scrollId, ex); } }
@Override public int doScoll() { rsp = client.prepareSearchScroll(scrollId()).setScroll(TimeValue.timeValueMinutes(keepTimeInMinutes)). execute().actionGet(); return rsp.getHits().hits().length; }
@Override public SearchResponse executeESScrollSearchNextRequest(SearchResponse scrollResp) { return client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(ES_SCROLL_KEEPALIVE)).execute() .actionGet(); }
@Override public void onResponse(final BulkResponse bulkResponse) { if (bulkResponse.hasFailures()) { throw new ReindexingException(bulkResponse .buildFailureMessage()); } client.prepareSearchScroll(scrollId).setScroll(scroll) .execute(ReindexingListener.this); }
void consumeBatches(ObjectOutputStream objectOutputStream, SearchResponse initialSearchResponse) throws IOException { SearchResponse batchSearchResponse = initialSearchResponse; while (writeSearchResponseToOutputStream(objectOutputStream, batchSearchResponse)) { String scrollId = batchSearchResponse.getScrollId(); batchSearchResponse = client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)).execute().actionGet(); } }
public KeyIterator(ElasticsearchIndex index) { this.index = index; MatchAllQueryBuilder query = QueryBuilders.matchAllQuery(); scrollResp = index.getClient().prepareSearch(index.getIndex()).setTypes(index.getType()) .addFields(new String[] {}).setSearchType(SearchType.SCAN) .setScroll(new TimeValue(ElasticsearchIndex.SCROLLTIMEOUT)).setQuery(query) .setSize(ElasticsearchIndex.SCROLLSIZE).execute().actionGet(); scrollResp = index.getClient().prepareSearchScroll(scrollResp.getScrollId()) .setScroll(new TimeValue(ElasticsearchIndex.SCROLLTIMEOUT)).execute().actionGet(); iterator = scrollResp.getHits().iterator(); }