/** * Delete documents using a query. Check what would be deleted first with a normal search query! * Elasticsearch once provided a native prepareDeleteByQuery method, but this was removed * in later versions. Instead, there is a plugin which iterates over search results, * see https://www.elastic.co/guide/en/elasticsearch/plugins/current/plugins-delete-by-query.html * We simulate the same behaviour here without the need of that plugin. * * @param q * @return delete document count */ public int deleteByQuery(String indexName, final QueryBuilder q) { Map<String, String> ids = new TreeMap<>(); // FIXME: deprecated, "will be removed in 3.0, you should do a regular scroll instead, ordered by `_doc`" @SuppressWarnings("deprecation") SearchResponse response = elasticsearchClient.prepareSearch(indexName).setSearchType(SearchType.SCAN) .setScroll(new TimeValue(60000)).setQuery(q).setSize(100).execute().actionGet(); while (true) { // accumulate the ids here, don't delete them right now to prevent an interference of the delete with the // scroll for (SearchHit hit : response.getHits().getHits()) { ids.put(hit.getId(), hit.getType()); } response = elasticsearchClient.prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000)) .execute().actionGet(); // termination if (response.getHits().getHits().length == 0) break; } return deleteBulk(indexName, ids); }
return new SqlElasticSearchRequestBuilder(new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE, (String) scrollHint.getParams()[0]).setScroll(new TimeValue((Integer) scrollHint.getParams()[1]))); this.request = new SearchRequestBuilder(client, SearchAction.INSTANCE); setIndicesAndTypes(); request.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC); request.setSize((Integer) scrollHint.getParams()[0]).setScroll(new TimeValue((Integer) scrollHint.getParams()[1])); } else { request.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
public List<String> findAllViewUuids() { SearchRequestBuilder esSearch = esClient.prepareSearch(ViewIndexDefinition.INDEX_TYPE_VIEW) .addSort("_doc", SortOrder.ASC) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) .setFetchSource(false) .setSize(100) .setQuery(matchAllQuery()); SearchResponse response = esSearch.get(); List<String> result = newArrayList(); while (true) { List<SearchHit> hits = newArrayList(response.getHits()); for (SearchHit hit : hits) { result.add(hit.getId()); } String scrollId = response.getScrollId(); response = esClient.prepareSearchScroll(scrollId) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) .get(); // Break condition: No hits are returned if (response.getHits().getHits().length == 0) { esClient.nativeClient().prepareClearScroll().addScrollId(scrollId).get(); break; } } return result; } }
@Override public boolean hasNext() { if (hits.isEmpty()) { SearchScrollRequestBuilder esRequest = esClient.prepareSearchScroll(scrollId) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); Collections.addAll(hits, esRequest.get().getHits().getHits()); } return !hits.isEmpty(); }
private List<SearchHit> scrollTillLimit(TableInJoinRequestBuilder tableInJoinRequest, Integer hintLimit) { SearchResponse scrollResp = scrollOneTimeWithMax(client,tableInJoinRequest); updateMetaSearchResults(scrollResp); List<SearchHit> hitsWithScan = new ArrayList<>(); int curentNumOfResults = 0; SearchHit[] hits = scrollResp.getHits().getHits(); if (hintLimit == null) hintLimit = MAX_RESULTS_FOR_FIRST_TABLE; while (hits.length != 0 && curentNumOfResults < hintLimit) { curentNumOfResults += hits.length; Collections.addAll(hitsWithScan, hits); if (curentNumOfResults >= MAX_RESULTS_FOR_FIRST_TABLE) { //todo: log or exception? System.out.println("too many results for first table, stoping at:" + curentNumOfResults); break; } scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); hits = scrollResp.getHits().getHits(); } return hitsWithScan; }
public void deleteAll() { Client client = getConnection().getClient(); TimeValue keepAlive = TimeValue.timeValueMinutes(10); SearchResponse response = client.prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(QueryBuilders.matchAllQuery()) .setSize(100) .setScroll(keepAlive) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); do { Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToNamedQuery) .forEach(this::delete); response = client.prepareSearchScroll(response.getScrollId()) .setScroll(keepAlive) .execute() .actionGet(); } while (response.getHits().getHits().length != 0); }
SearchResponse searchResponse = client.prepareSearch() .setIndices(pubSubIndexName) .setTypes("publish") .setSearchType(SearchType.SCAN) .setScroll(scrollTimeout) .setQuery(queryBuilder) long totalHits = searchResponse.getHits().getTotalHits(); boolean zero = totalHits == 0L; if (zero) { searchResponse = client.prepareSearchScroll(searchResponse.getScrollId()) .setScroll(scrollTimeout) .execute().actionGet(); for (SearchHit hit : searchResponse.getHits()) { Long timestamp = (Long) hit.field("timestamp").getValues().get(0); Map<String, Object> data = hit.field("data").getValue(); channel.write(new NettyInteractiveResponse("message", createPublishMessage(timestamp, data)).response()); if (searchResponse.getHits().hits().length == 0) { break;
int docCount = Ints.saturatedCast(countResponse.getHits().totalHits()); final SearchResponse scrollResponse = client.prepareSearch() .setSearchType(SearchType.SCAN) .setScroll(new TimeValue(1L, TimeUnit.MINUTES)) .setQuery(QueryBuilders.matchAllQuery()) .setSize(docCount) .actionGet(); final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); while (true) { final SearchResponse searchResponse = client.prepareSearchScroll(scrollResponse.getScrollId()) .setScroll(new TimeValue(1L, TimeUnit.MINUTES)) .execute() .actionGet(); for (SearchHit hit : searchResponse.getHits().getHits()) { bulkRequestBuilder.add(client.prepareDelete(hit.index(), hit.type(), hit.id())); if (searchResponse.getHits().getHits().length == 0) { break;
@Override public Set<String> getFolderNames() throws IOException, MessagingException { createIndexIfNotExists(); client.admin().indices().refresh(new RefreshRequest()).actionGet(); final HashSet<String> uids = new HashSet<String>(); SearchResponse scrollResp = client.prepareSearch().setIndices(index).setTypes(type).setSearchType(SearchType.SCAN) .setQuery(QueryBuilders.matchAllQuery()).addField("folderFullName").setScroll(new TimeValue(1000)).setSize(1000).execute() .actionGet(); while (true) { scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(1000)).execute().actionGet(); boolean hitsRead = false; for (final SearchHit hit : scrollResp.getHits()) { hitsRead = true; uids.add((String) hit.getFields().get("folderFullName").getValue()); } if (!hitsRead) { break; } } if (logger.isDebugEnabled()) { logger.debug("Currently locally stored folders: {}", uids); } return uids; }
while (true) { if (response == null) { response = client.prepareSearch(userIndex) .setTypes(userType) .setScroll(scrollKeepAlive) .setQuery(QueryBuilders.boolQuery() .must(userQueryBuilder) .filter(getLastAccessedFilterQuery())) .addFields(userIdField).setSize(scrollSize) .execute().actionGet(); long totalHits = response.getHits().getTotalHits(); if (totalHits > Integer.MAX_VALUE) { logger.warn("The number of users is {} > {}.", totalHits, } else { response = client .prepareSearchScroll(response.getScrollId()) .setScroll(scrollKeepAlive).execute().actionGet(); if (response.getHits().getHits().length == 0) { break;
while (true) { if (response == null) { response = client.prepareSearch(preferenceIndex) .setTypes(preferenceType).setScroll(scrollKeepAlive) .setQuery(QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(userIdField, userID)) } else { response = client .prepareSearchScroll(response.getScrollId()) .setScroll(scrollKeepAlive).execute().actionGet(); final int size = response.getHits().getHits().length; if (size == 0) { break; final BulkRequestBuilder bulkRequest = client.prepareBulk(); for (final SearchHit hit : response.getHits()) { bulkRequest.add(client.prepareDelete(hit.getIndex(), hit.getType(), hit.getId()));
try { scrollResponse = esClientProvider.getClient().prepareSearch(typeDescriptor.getIndex()) .setTypes(typeDescriptor.getType()) .setFetchSource(false) .addSort(DOC, SortOrder.ASC) .setVersion(true) .setScroll(scrollTimeout) if (scrollResponse.getHits().getHits().length == 0) { break; for (SearchHit hit : scrollResponse.getHits().hits()) { DeleteRequest delete = new DeleteRequest().index(hit.index()) .type(hit.type()) .id(hit.id()); bulkRequest.add(delete); esClientProvider.getClient().bulk(bulkRequest).actionGet(queryTimeout); scrollResponse = esClientProvider.getClient().prepareSearchScroll(scrollResponse.getScrollId()) .setScroll(scrollTimeout) .execute() .actionGet(queryTimeout);
boolean finishedScrolling; if (hintLimit != null && hintLimit < MAX_RESULTS_ON_ONE_FETCH) { searchResponse = secondTableRequest.getRequestBuilder().setSize(hintLimit).get(); finishedScrolling = true; } else { searchResponse = secondTableRequest.getRequestBuilder() .setScroll(new TimeValue(60000)) .setSize(MAX_RESULTS_ON_ONE_FETCH).get(); int fetchedSoFarFromSecondTable = 0; while (!limitReached) { SearchHit[] secondTableHits = searchResponse.getHits().getHits(); fetchedSoFarFromSecondTable += secondTableHits.length; for (SearchHit secondTableHit : secondTableHits) { List<SearchHit> searchHits = searchHitsResult.getSearchHits(); for (SearchHit matchingHit : searchHits) { String combinedId = matchingHit.getId() + "|" + secondTableHit.getId(); copyMaps(copiedSource,secondTableHit.getSourceAsMap()); onlyReturnedFields(copiedSource, secondTableRequest.getReturnedFields(),secondTableRequest.getOriginalSelect().isSelectAll()); searchResponse = client.prepareSearchScroll(searchResponse.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); } else break; } else {
/** * Get all the indexed documents (no paginated results). Results are not sorted. */ public List<SearchHit> getDocuments(IndexType indexType) { SearchRequestBuilder req = SHARED_NODE.client().prepareSearch(indexType.getIndex()).setTypes(indexType.getType()).setQuery(matchAllQuery()); EsUtils.optimizeScrollRequest(req); req.setScroll(new TimeValue(60000)) .setSize(100); SearchResponse response = req.get(); List<SearchHit> result = newArrayList(); while (true) { Iterables.addAll(result, response.getHits()); response = SHARED_NODE.client().prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); // Break condition: No hits are returned if (response.getHits().getHits().length == 0) { break; } } return result; }
public void addDeletion(SearchRequestBuilder searchRequest) { .addSort("_doc", SortOrder.ASC) .setScroll(TimeValue.timeValueMinutes(5)) .setSize(100) .setFetchSource(false); SearchHit[] hits = searchResponse.getHits().getHits(); for (SearchHit hit : hits) { SearchHitField routing = hit.getField("_routing"); DeleteRequestBuilder deleteRequestBuilder = client.prepareDelete(hit.getIndex(), hit.getType(), hit.getId()); if (routing != null) { deleteRequestBuilder.setRouting(routing.getValue()); String scrollId = searchResponse.getScrollId(); if (scrollId == null) { break; searchResponse = client.prepareSearchScroll(scrollId).setScroll(TimeValue.timeValueMinutes(5)).get(); if (hits.length == 0) { client.nativeClient().prepareClearScroll().addScrollId(scrollId).get(); break;
while (true) { if (response == null) { response = client.prepareSearch(index).setTypes(type) .setScroll(new TimeValue(keepAlive.longValue())) .setQuery(QueryBuilders.queryStringQuery(userQuery)) .addField(fieldName).setSize(size.intValue()).execute() .actionGet(); } else { response = client.prepareSearchScroll(response.getScrollId()) .setScroll(new TimeValue(keepAlive.longValue())) .execute().actionGet(); final SearchHits hits = response.getHits(); if (targetIDs == null) { targetIDs = new long[(int) hits.getTotalHits()]; if (logger.isDebugEnabled()) { logger.debug("{} users are found by {}", hits.getTotalHits(), userQuery); if (hits.getHits().length == 0) { break; final SearchHitField searchHitField = hit.getFields().get( fieldName); final Number value = searchHitField.getValue();
SearchRequestBuilder builder = client.get().prepareSearch(searchableIndexes) .setTypes(TYPE) .setQuery(query) .setScroll(new TimeValue(1, TimeUnit.MINUTES)) .setSize(100) .setProfile(profile); response = builder.execute().actionGet(); iterator = Arrays.asList(response.getHits().getHits()).iterator(); noMoreHits = !iterator.hasNext(); SearchScrollRequestBuilder builder = client.get().prepareSearchScroll(response.getScrollId()) .setScroll(new TimeValue(1, TimeUnit.MINUTES)); response = builder.execute().actionGet(); iterator = Arrays.asList(response.getHits().getHits()).iterator(); noMoreHits = !iterator.hasNext();
void dumpSave(Client client, Builder builder) throws IOException { TimeValue scrollDuration = TimeValue.timeValueSeconds(30); SearchResponse searchResponse = client.prepareSearch(builder.indices()) .setTypes(builder.types()).setSearchType(SearchType.SCAN) .setQuery(builder.query()).setSize(100).setScroll(scrollDuration).execute().actionGet(); OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(builder.path(), true), builder.charset()); int i = 0; try { while (true) { searchResponse = client.prepareSearchScroll(searchResponse.getScrollId()) .setScroll(scrollDuration).execute().actionGet(); if (searchResponse.getHits().hits().length == 0) { break; } for (SearchHit hit : searchResponse.getHits()) { Document document = Document.fromSource(hit.sourceAsString(), hit.index(), hit.type(), hit.id()); writer.write(document.getDump()); i++; writer.write('\n'); } } } finally { writer.close(); console.println("Saved " + i + " documents to " + builder.path()); } }
search.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);//查询方式(共4种查询方式) search.setSource(searchSourceBuilder); search.setSize(size); search.setScroll(TimeValue.timeValueSeconds(scrollSeconds)); if (count.intValue() == 0) { scrollResponse = search.execute().actionGet(); count = scrollResponse.getHits().getTotalHits(); } else { scrollResponse = client.prepareSearchScroll(scrollResponse.getScrollId()) .setScroll(TimeValue.timeValueSeconds(scrollSeconds)) .execute().actionGet(); SearchHits hits = scrollResponse.getHits(); sum += hits.getHits().length; if (null != hits && hits.getHits().length > 0) { for (SearchHit hit : hits) { String source = hit.getSourceAsString(); EsLogUtils.info("查询到: {}", source); list.add(EsJsonUtils.getObject(source, clazz));
public KeyIterator(ElasticsearchIndex index) { this.index = index; MatchAllQueryBuilder query = QueryBuilders.matchAllQuery(); scrollResp = index.getClient().prepareSearch(index.getIndex()).setTypes(index.getType()) .addFields(new String[] {}).setSearchType(SearchType.SCAN) .setScroll(new TimeValue(ElasticsearchIndex.SCROLLTIMEOUT)).setQuery(query) .setSize(ElasticsearchIndex.SCROLLSIZE).execute().actionGet(); scrollResp = index.getClient().prepareSearchScroll(scrollResp.getScrollId()) .setScroll(new TimeValue(ElasticsearchIndex.SCROLLTIMEOUT)).execute().actionGet(); iterator = scrollResp.getHits().iterator(); }