/** * Delete documents using a query. Check what would be deleted first with a normal search query! * Elasticsearch once provided a native prepareDeleteByQuery method, but this was removed * in later versions. Instead, there is a plugin which iterates over search results, * see https://www.elastic.co/guide/en/elasticsearch/plugins/current/plugins-delete-by-query.html * We simulate the same behaviour here without the need of that plugin. * * @param q * @return delete document count */ public int deleteByQuery(String indexName, final QueryBuilder q) { Map<String, String> ids = new TreeMap<>(); // FIXME: deprecated, "will be removed in 3.0, you should do a regular scroll instead, ordered by `_doc`" @SuppressWarnings("deprecation") SearchResponse response = elasticsearchClient.prepareSearch(indexName).setSearchType(SearchType.SCAN) .setScroll(new TimeValue(60000)).setQuery(q).setSize(100).execute().actionGet(); while (true) { // accumulate the ids here, don't delete them right now to prevent an interference of the delete with the // scroll for (SearchHit hit : response.getHits().getHits()) { ids.put(hit.getId(), hit.getType()); } response = elasticsearchClient.prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000)) .execute().actionGet(); // termination if (response.getHits().getHits().length == 0) break; } return deleteBulk(indexName, ids); }
public Map<String, Object> query(final String indexName, final String fieldKey, final String fieldValue) { if (fieldKey == null || fieldValue.length() == 0) return null; // prepare request BoolQueryBuilder query = QueryBuilders.boolQuery(); query.filter(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery(fieldKey, fieldValue))); SearchRequestBuilder request = elasticsearchClient.prepareSearch(indexName) .setSearchType(SearchType.QUERY_THEN_FETCH) .setQuery(query) .setFrom(0) .setSize(1) .setTerminateAfter(1); // get response SearchResponse response = request.execute().actionGet(); // evaluate search result SearchHit[] hits = response.getHits().getHits(); if (hits.length == 0) return null; assert hits.length == 1; Map<String, Object> map = hits[0].getSource(); return map; }
QueryBuilder queryBuilder = expression.getFilterBuilder(); BoolQueryBuilder filterQuery = QueryBuilders.boolQuery().must(queryBuilder); QueryStringQueryBuilder stringQuery = QueryBuilders.queryStringQuery("*"); BoolQueryBuilder fq = QueryBuilders.boolQuery().must(stringQuery).must(filterQuery); final SearchRequestBuilder srb = elasticSearchClient.prepareSearch(logIndexPrefix + "*") .setQuery(fq) .setTypes(LOG_DOC_TYPE) .addSort(sortBuilder); SearchResponse response = srb.execute().actionGet(); return Arrays.stream(response.getHits().getHits()) .map(hit -> { String source = hit.getSourceAsString(); try { return objectMapper.readValue(source, TaskExecLog.class);
protected SearchHit createUnmachedResult( List<Field> secondTableReturnedFields, int docId, String t1Alias, String t2Alias, SearchHit hit) { String unmatchedId = hit.getId() + "|0"; Text unamatchedType = new Text(hit.getType() + "|null"); SearchHit searchHit = new SearchHit(docId, unmatchedId, unamatchedType, hit.getFields()); searchHit.sourceRef(hit.getSourceRef()); searchHit.getSourceAsMap().clear(); searchHit.getSourceAsMap().putAll(hit.getSourceAsMap()); Map<String,Object> emptySecondTableHitSource = createNullsSource(secondTableReturnedFields); mergeSourceAndAddAliases(emptySecondTableHitSource, searchHit,t1Alias,t2Alias); return searchHit; }
public SearchResult(SearchResponse resp) { SearchHits hits = resp.getHits(); this.total = hits.getTotalHits(); results = new ArrayList<>(hits.getHits().length); for (SearchHit searchHit : hits.getHits()) { if (searchHit.getSourceAsMap() != null) { results.add(searchHit.getSourceAsMap()); } else if (searchHit.getFields() != null) { Map<String, DocumentField> fields = searchHit.getFields(); results.add(toFieldsMap(fields)); } } }
@Override public boolean advanceNextPosition() { if (!searchHits.hasNext()) { return false; } SearchHit hit = searchHits.next(); fields = new ArrayList<>(Collections.nCopies(columnHandles.size(), null)); setFieldIfExists("_id", hit.getId()); setFieldIfExists("_index", hit.getIndex()); extractFromSource(hit); if (hit.getSourceRef() != null) { totalBytes += hit.getSourceRef().length(); } return true; }
@Override public List<String> searchArchivableWorkflows(String indexName, long archiveTtlDays) { QueryBuilder q = QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("status", "COMPLETED")) .should(QueryBuilders.termQuery("status", "FAILED")) .mustNot(QueryBuilders.existsQuery("archived")) .minimumShouldMatch(1); SearchRequestBuilder s = elasticSearchClient.prepareSearch(indexName) .setTypes("workflow") .setQuery(q) .addSort("endTime", SortOrder.ASC) .setSize(archiveSearchBatchSize); SearchResponse response = s.execute().actionGet(); SearchHits hits = response.getHits(); logger.info("Archive search totalHits - {}", hits.getTotalHits()); return Arrays.stream(hits.getHits()) .map(hit -> hit.getId()) .collect(Collectors.toCollection(LinkedList::new)); }
private Query(final String indexName, QueryBuilder queryBuilder, String order_field, int resultCount) { //TODO: sort data using order_field // prepare request SearchRequestBuilder request = elasticsearchClient.prepareSearch(indexName) .setSearchType(SearchType.QUERY_THEN_FETCH) .setQuery(queryBuilder) .setFrom(0) .setSize(resultCount); request.clearRescorers(); // get response SearchResponse response = request.execute().actionGet(); hitCount = (int) response.getHits().getTotalHits(); // evaluate search result SearchHit[] hits = response.getHits().getHits(); this.result = new ArrayList<Map<String, Object>>(hitCount); for (SearchHit hit: hits) { Map<String, Object> map = hit.getSource(); this.result.add(map); } }
@Override public List<String> searchRecentRunningWorkflows(int lastModifiedHoursAgoFrom, int lastModifiedHoursAgoTo) { DateTime dateTime = new DateTime(); QueryBuilder q = QueryBuilders.boolQuery() .must(QueryBuilders.rangeQuery("updateTime") .gt(dateTime.minusHours(lastModifiedHoursAgoFrom))) .must(QueryBuilders.rangeQuery("updateTime") .lt(dateTime.minusHours(lastModifiedHoursAgoTo))) .must(QueryBuilders.termQuery("status", "RUNNING")); SearchRequestBuilder s = elasticSearchClient.prepareSearch(indexName) .setTypes("workflow") .setQuery(q) .setSize(5000) .addSort("updateTime", SortOrder.ASC); SearchResponse response = s.execute().actionGet(); return StreamSupport.stream(response.getHits().spliterator(), false) .map(hit -> hit.getId()) .collect(Collectors.toCollection(LinkedList::new)); }
@Override public Iterable<RawQuery.Result<String>> query(RawQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException { SearchRequestBuilder srb = client.prepareSearch(indexName); srb.setTypes(query.getStore()); srb.setQuery(QueryBuilders.queryStringQuery(query.getQuery())); srb.setFrom(query.getOffset()); if (query.hasLimit()) srb.setSize(query.getLimit()); else srb.setSize(maxResultsSize); srb.setNoFields(); //srb.setExplain(true); SearchResponse response = srb.execute().actionGet(); log.debug("Executed query [{}] in {} ms", query.getQuery(), response.getTookInMillis()); SearchHits hits = response.getHits(); if (!query.hasLimit() && hits.totalHits() >= maxResultsSize) log.warn("Query result set truncated to first [{}] elements for query: {}", maxResultsSize, query); List<RawQuery.Result<String>> result = new ArrayList<RawQuery.Result<String>>(hits.hits().length); for (SearchHit hit : hits) { result.add(new RawQuery.Result<String>(hit.id(),hit.getScore())); } return result; }
BoolQueryBuilder suggest = QueryBuilders.boolQuery(); if (q != null && q.length() > 0) { suggest.should(QueryBuilders.fuzzyQuery(fieldName, q).fuzziness(Fuzziness.fromEdits(2))); suggest.should(QueryBuilders.moreLikeThisQuery(fieldName).like(q)); suggest.should(QueryBuilders.matchPhrasePrefixQuery(fieldName, q)); if (q.indexOf('*') >= 0 || q.indexOf('?') >= 0) suggest.should(QueryBuilders.wildcardQuery(fieldName, q)); SearchRequestBuilder request = this.elasticsearchClient.prepareSearch(indexName) .setSearchType(SearchType.QUERY_THEN_FETCH) .setQuery(query) .setFrom(0) .setSize(resultCount) .addSort( SearchResponse response = request.execute().actionGet(); SearchHits rhits = response.getHits(); SearchHit[] hits = rhits.getHits(); for (SearchHit hit: hits) { Map<String, Object> map = hit.getSource(); result.add(map); result.setHits(rhits.getTotalHits());
public void append4Update(BulkRequestBuilder bulkRequestBuilder, ESMapping mapping, Object pkVal, Map<String, Object> esFieldData) { if (mapping.get_id() != null) { bulkRequestBuilder .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), pkVal.toString()) .setDoc(esFieldData)); } else { SearchResponse response = transportClient.prepareSearch(mapping.get_index()) .setTypes(mapping.get_type()) .setQuery(QueryBuilders.termQuery(mapping.getPk(), pkVal)) .setSize(MAX_BATCH_SIZE) .get(); for (SearchHit hit : response.getHits()) { bulkRequestBuilder .add(transportClient.prepareUpdate(mapping.get_index(), mapping.get_type(), hit.getId()) .setDoc(esFieldData)); } } }
Collection<Record> records = new ArrayList<Record>(); BoolQueryBuilder bqb = QueryBuilders.boolQuery(); continue; QueryStringQueryBuilder qsqb = QueryBuilders .queryString(queryString.toString().trim()) .defaultField(propName).boost(boostFactor); bqb = required ? bqb.must(qsqb) : bqb.should(qsqb); SearchResponse response = this.client.prepareSearch(this.indexName) .setTypes(this.indexType) .setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(bqb) .setSize(this.maxSearchHits).execute().actionGet(); SearchHit[] results = response.getHits().getHits(); for (SearchHit hit : results) { records.add(this.readFromSource(hit.getId(), hit.getSource()));
public Map<String, Object> query(final String indexName, final String typeName, final QueryBuilder query) { SearchRequestBuilder request = elasticsearchClient.prepareSearch(indexName); if (typeName != null) request.setTypes(typeName); request .setSearchType(SearchType.QUERY_THEN_FETCH) .setQuery(query) .setFrom(0) .setSize(1).setTerminateAfter(1); // get response SearchResponse response = request.execute().actionGet(); // evaluate search result //long totalHitCount = response.getHits().getTotalHits(); SearchHit[] hits = response.getHits().getHits(); if (hits.length == 0) return null; assert hits.length == 1; Map<String, Object> map = hits[0].getSourceAsMap(); if (!map.containsKey("id")) map.put("id", hits[0].getId()); if (!map.containsKey("type")) map.put("type", hits[0].getType()); return map; }
private void deleteBulkRequest(String objectId, String index, String type, String routing, String parent) { if (logger.isTraceEnabled()) { logger.trace("bulkDeleteRequest - objectId: {} - index: {} - type: {} - routing: {} - parent: {}", objectId, index, type, routing, parent); } if (definition.getParentTypes() != null && definition.getParentTypes().contains(type)) { QueryBuilder builder = QueryBuilders.hasParentQuery(type, QueryBuilders.termQuery(MongoDBRiver.MONGODB_ID_FIELD, objectId)); SearchResponse response = esClient.prepareSearch(index).setQuery(builder).setRouting(routing) .addField(MongoDBRiver.MONGODB_ID_FIELD).execute().actionGet(); for (SearchHit hit : response.getHits().getHits()) { getBulkProcessor(index, hit.getType()).deleteBulkRequest(hit.getId(), routing, objectId); } } getBulkProcessor(index, type).deleteBulkRequest(objectId, routing, parent); }
Vector<HashMap<String, ByteIterator>> result) { try { final RangeQueryBuilder rangeQuery = rangeQuery("_id").gte(startkey); final SearchResponse response = client.prepareSearch(indexKey) .setTypes(table) .setQuery(rangeQuery) .setSize(recordcount) .execute() .actionGet(); for (SearchHit hit : response.getHits()) { entry = new HashMap<>(fields.size()); for (String field : fields) { entry.put(field, new StringByteIterator((String) hit.getSource().get(field)));
public List<String> findAllViewUuids() { SearchRequestBuilder esSearch = esClient.prepareSearch(ViewIndexDefinition.INDEX_TYPE_VIEW) .addSort("_doc", SortOrder.ASC) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) .setFetchSource(false) .setSize(100) .setQuery(matchAllQuery()); SearchResponse response = esSearch.get(); List<String> result = newArrayList(); while (true) { List<SearchHit> hits = newArrayList(response.getHits()); for (SearchHit hit : hits) { result.add(hit.getId()); } String scrollId = response.getScrollId(); response = esClient.prepareSearchScroll(scrollId) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)) .get(); // Break condition: No hits are returned if (response.getHits().getHits().length == 0) { esClient.nativeClient().prepareClearScroll().addScrollId(scrollId).get(); break; } } return result; } }
/** * Returns the active users (at most 3) who are associated to the given SCM account. This method can be used * to detect user conflicts. */ public List<UserDoc> getAtMostThreeActiveUsersForScmAccount(String scmAccount, String organizationUuid) { List<UserDoc> result = new ArrayList<>(); if (!StringUtils.isEmpty(scmAccount)) { SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.INDEX_TYPE_USER) .setQuery(boolQuery().must(matchAllQuery()).filter( boolQuery() .must(termQuery(FIELD_ACTIVE, true)) .must(termQuery(FIELD_ORGANIZATION_UUIDS, organizationUuid)) .should(termQuery(FIELD_LOGIN, scmAccount)) .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_EMAIL), scmAccount)) .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_SCM_ACCOUNTS), scmAccount)))) .setSize(3); for (SearchHit hit : request.get().getHits().getHits()) { result.add(new UserDoc(hit.getSourceAsMap())); } } return result; }
public List<Map<String, Object>> getAllIndexedDocs(String index, String sourceType, String subMessage) throws IOException { getClient().admin().indices().refresh(new RefreshRequest()); SearchResponse response = getClient().prepareSearch(index) .setTypes(sourceType) .setFrom(0) .setSize(1000) .execute().actionGet(); List<Map<String, Object>> ret = new ArrayList<Map<String, Object>>(); for (SearchHit hit : response.getHits()) { Object o = null; if (subMessage == null) { o = hit.getSource(); } else { o = hit.getSource().get(subMessage); } ret.add((Map<String, Object>) (o)); } return ret; }
/** * 分页查询 * @throws Exception */ @Test public void searchPaging()throws Exception{ SearchRequestBuilder srb=client.prepareSearch("film").setTypes("dongzuo"); SearchResponse sr=srb.setQuery(QueryBuilders.matchAllQuery()).setFrom(1).setSize(2).execute().actionGet(); // 查询所有 SearchHits hits=sr.getHits(); for(SearchHit hit:hits){ System.out.println(hit.getSourceAsString()); } }