@Override public List<String> searchArchivableWorkflows(String indexName, long archiveTtlDays) { QueryBuilder q = QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("status", "COMPLETED")) .should(QueryBuilders.termQuery("status", "FAILED")) .mustNot(QueryBuilders.existsQuery("archived")) .minimumShouldMatch(1); SearchRequestBuilder s = elasticSearchClient.prepareSearch(indexName) .setTypes("workflow") .setQuery(q) .addSort("endTime", SortOrder.ASC) .setSize(archiveSearchBatchSize); SearchResponse response = s.execute().actionGet(); SearchHits hits = response.getHits(); logger.info("Archive search totalHits - {}", hits.getTotalHits()); return Arrays.stream(hits.getHits()) .map(hit -> hit.getId()) .collect(Collectors.toCollection(LinkedList::new)); }
/** * Returns the active users (at most 3) who are associated to the given SCM account. This method can be used * to detect user conflicts. */ public List<UserDoc> getAtMostThreeActiveUsersForScmAccount(String scmAccount, String organizationUuid) { List<UserDoc> result = new ArrayList<>(); if (!StringUtils.isEmpty(scmAccount)) { SearchRequestBuilder request = esClient.prepareSearch(UserIndexDefinition.INDEX_TYPE_USER) .setQuery(boolQuery().must(matchAllQuery()).filter( boolQuery() .must(termQuery(FIELD_ACTIVE, true)) .must(termQuery(FIELD_ORGANIZATION_UUIDS, organizationUuid)) .should(termQuery(FIELD_LOGIN, scmAccount)) .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_EMAIL), scmAccount)) .should(matchQuery(SORTABLE_ANALYZER.subField(FIELD_SCM_ACCOUNTS), scmAccount)))) .setSize(3); for (SearchHit hit : request.get().getHits().getHits()) { result.add(new UserDoc(hit.getSourceAsMap())); } } return result; }
@Override public List<String> searchRecentRunningWorkflows(int lastModifiedHoursAgoFrom, int lastModifiedHoursAgoTo) { DateTime dateTime = new DateTime(); QueryBuilder q = QueryBuilders.boolQuery() .must(QueryBuilders.rangeQuery("updateTime") .gt(dateTime.minusHours(lastModifiedHoursAgoFrom))) .must(QueryBuilders.rangeQuery("updateTime") .lt(dateTime.minusHours(lastModifiedHoursAgoTo))) .must(QueryBuilders.termQuery("status", "RUNNING")); SearchRequestBuilder s = elasticSearchClient.prepareSearch(indexName) .setTypes("workflow") .setQuery(q) .setSize(5000) .addSort("updateTime", SortOrder.ASC); SearchResponse response = s.execute().actionGet(); return StreamSupport.stream(response.getHits().spliterator(), false) .map(hit -> hit.getId()) .collect(Collectors.toCollection(LinkedList::new)); }
private static AbstractAggregationBuilder createNoDataFacet(String metricKey) { return AggregationBuilders.filter( "no_data_" + metricKey, boolQuery().mustNot(nestedQuery(FIELD_MEASURES, termQuery(FIELD_MEASURES_KEY, metricKey), ScoreMode.Avg))); }
private static BoolQueryBuilder addTermFilter(BoolQueryBuilder filter, String field, @Nullable Collection<String> values) { if (isNotEmpty(values)) { BoolQueryBuilder valuesFilter = boolQuery(); for (String value : values) { QueryBuilder valueFilter = QueryBuilders.termQuery(field, value); valuesFilter.should(valueFilter); } filter.must(valuesFilter); } return filter; }
@Nullable private QueryBuilder standardFilters(TimeRange range, String filter) { BoolQueryBuilder bfb = null; if (range != null) { bfb = QueryBuilders.boolQuery(); bfb.must(IndexHelper.getTimestampRangeFilter(range)); } // Not creating a filter for a "*" value because an empty filter used to be submitted that way. if (!isNullOrEmpty(filter) && !"*".equals(filter)) { if (bfb == null) { bfb = QueryBuilders.boolQuery(); } bfb.must(queryStringQuery(filter)); } return bfb; }
@Override public List<EventExecution> getEventExecutions(String event) { try { Expression expression = Expression.fromString("event='" + event + "'"); QueryBuilder queryBuilder = expression.getFilterBuilder(); BoolQueryBuilder filterQuery = QueryBuilders.boolQuery().must(queryBuilder); QueryStringQueryBuilder stringQuery = QueryBuilders.queryStringQuery("*"); BoolQueryBuilder fq = QueryBuilders.boolQuery().must(stringQuery).must(filterQuery); final SearchRequestBuilder srb = elasticSearchClient.prepareSearch(logIndexPrefix + "*") .setQuery(fq).setTypes(EVENT_DOC_TYPE) .addSort(SortBuilders.fieldSort("created") .order(SortOrder.ASC)); return mapEventExecutionsResponse(srb.execute().actionGet()); } catch (Exception e) { logger.error("Failed to get executions for event: {}", event, e); throw new ApplicationException(Code.BACKEND_ERROR, e.getMessage(), e); } }
public List<Map<String, Object>> queryWithConstraints(final String indexName, final String fieldName, final String fieldValue, final Map<String, String> constraints, boolean latest) throws IOException { SearchRequestBuilder request = this.elasticsearchClient.prepareSearch(indexName) .setSearchType(SearchType.QUERY_THEN_FETCH) .setFrom(0); BoolQueryBuilder bFilter = QueryBuilders.boolQuery(); bFilter.filter(QueryBuilders.constantScoreQuery(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery(fieldName, fieldValue)))); for (Object o : constraints.entrySet()) { @SuppressWarnings("rawtypes") Map.Entry entry = (Map.Entry) o; bFilter.filter(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery((String) entry.getKey(), ((String) entry.getValue()).toLowerCase()))); } request.setQuery(bFilter); // get response SearchResponse response = request.execute().actionGet(); // evaluate search result ArrayList<Map<String, Object>> result = new ArrayList<Map<String, Object>>(); SearchHit[] hits = response.getHits().getHits(); for (SearchHit hit: hits) { Map<String, Object> map = hit.getSource(); result.add(map); } return result; }
BoolQueryBuilder suggest = QueryBuilders.boolQuery(); if (q != null && q.length() > 0) { suggest.should(QueryBuilders.fuzzyQuery(fieldName, q).fuzziness(Fuzziness.fromEdits(2))); suggest.should(QueryBuilders.moreLikeThisQuery(fieldName).like(q)); suggest.should(QueryBuilders.matchPhrasePrefixQuery(fieldName, q)); if (q.indexOf('*') >= 0 || q.indexOf('?') >= 0) suggest.should(QueryBuilders.wildcardQuery(fieldName, q)); suggest.minimumNumberShouldMatch(1); query = QueryBuilders.boolQuery(); if (q.length() > 0) query.filter(suggest); RangeQueryBuilder rangeQuery = QueryBuilders.rangeQuery(range_field); if (since != null) rangeQuery.from(since).includeLower(true); if (until != null) rangeQuery.to(until).includeUpper(true); SearchResponse response = request.execute().actionGet(); SearchHits rhits = response.getHits(); SearchHit[] hits = rhits.getHits(); for (SearchHit hit: hits) { Map<String, Object> map = hit.getSource(); result.add(map); result.setHits(rhits.getTotalHits());
private SearchResult<String> search(String indexName, String structuredQuery, int start, int size, List<String> sortOptions, String freeTextQuery, String docType) { try { QueryBuilder queryBuilder = QueryBuilders.matchAllQuery(); if (StringUtils.isNotEmpty(structuredQuery)) { Expression expression = Expression.fromString(structuredQuery); queryBuilder = expression.getFilterBuilder(); } BoolQueryBuilder filterQuery = QueryBuilders.boolQuery().must(queryBuilder); QueryStringQueryBuilder stringQuery = QueryBuilders.queryStringQuery(freeTextQuery); BoolQueryBuilder fq = QueryBuilders.boolQuery().must(stringQuery).must(filterQuery); final SearchRequestBuilder srb = elasticSearchClient.prepareSearch(indexName) .setQuery(fq) .setTypes(docType) .storedFields("_id") .setFrom(start) .setSize(size); if (sortOptions != null) { sortOptions.forEach(sortOption -> addSortOptionToSearchRequest(srb, sortOption)); } SearchResponse response = srb.get(); LinkedList<String> result = StreamSupport.stream(response.getHits().spliterator(), false) .map(SearchHit::getId) .collect(Collectors.toCollection(LinkedList::new)); long count = response.getHits().getTotalHits(); return new SearchResult<String>(count, result); } catch (ParserException e) { throw new ApplicationException(Code.BACKEND_ERROR, e.getMessage(), e); } }
public long count(final String index, final String histogram_timefield, final long millis) { try { SearchResponse response = elasticsearchClient.prepareSearch(index) .setSize(0) .setQuery(millis <= 0 ? QueryBuilders.constantScoreQuery(QueryBuilders.matchAllQuery()) : QueryBuilders.rangeQuery(histogram_timefield).from(new Date(System.currentTimeMillis() - millis))) .execute() .actionGet(); return response.getHits().getTotalHits(); } catch (Throwable e) { DAO.severe(e); return 0; } }
@Override public List<String> query(IndexQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException { SearchRequestBuilder srb = client.prepareSearch(indexName); srb.setTypes(query.getStore()); srb.setQuery(QueryBuilders.matchAllQuery()); srb.setPostFilter(getFilter(query.getCondition(),informations.get(query.getStore()))); if (!query.getOrder().isEmpty()) { List<IndexQuery.OrderEntry> orders = query.getOrder(); SearchResponse response = srb.execute().actionGet(); log.debug("Executed query [{}] in {} ms", query.getCondition(), response.getTookInMillis()); SearchHits hits = response.getHits(); if (!query.hasLimit() && hits.totalHits() >= maxResultsSize) log.warn("Query result set truncated to first [{}] elements for query: {}", maxResultsSize, query); List<String> result = new ArrayList<String>(hits.hits().length); for (SearchHit hit : hits) { result.add(hit.id());
@Override public Iterable<RawQuery.Result<String>> query(RawQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException { SearchRequestBuilder srb = client.prepareSearch(indexName); srb.setTypes(query.getStore()); srb.setQuery(QueryBuilders.queryStringQuery(query.getQuery())); srb.setFrom(query.getOffset()); if (query.hasLimit()) srb.setSize(query.getLimit()); else srb.setSize(maxResultsSize); srb.setNoFields(); //srb.setExplain(true); SearchResponse response = srb.execute().actionGet(); log.debug("Executed query [{}] in {} ms", query.getQuery(), response.getTookInMillis()); SearchHits hits = response.getHits(); if (!query.hasLimit() && hits.totalHits() >= maxResultsSize) log.warn("Query result set truncated to first [{}] elements for query: {}", maxResultsSize, query); List<RawQuery.Result<String>> result = new ArrayList<RawQuery.Result<String>>(hits.hits().length); for (SearchHit hit : hits) { result.add(new RawQuery.Result<String>(hit.id(),hit.getScore())); } return result; }
/** * Get all the indexed documents (no paginated results). Results are not sorted. */ public List<SearchHit> getDocuments(IndexType indexType) { SearchRequestBuilder req = SHARED_NODE.client().prepareSearch(indexType.getIndex()).setTypes(indexType.getType()).setQuery(matchAllQuery()); EsUtils.optimizeScrollRequest(req); req.setScroll(new TimeValue(60000)) .setSize(100); SearchResponse response = req.get(); List<SearchHit> result = newArrayList(); while (true) { Iterables.addAll(result, response.getHits()); response = SHARED_NODE.client().prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); // Break condition: No hits are returned if (response.getHits().getHits().length == 0) { break; } } return result; }
private void assertMemberInIndex(String organizationUuid, UserDto user, boolean isMember) { SearchRequestBuilder request = es.client().prepareSearch(UserIndexDefinition.INDEX_TYPE_USER) .setQuery(boolQuery() .must(termQuery(FIELD_ORGANIZATION_UUIDS, organizationUuid)) .must(termQuery(FIELD_UUID, user.getUuid()))); if (isMember) { assertThat(request.get().getHits().getHits()).hasSize(1); } else { assertThat(request.get().getHits().getHits()).isEmpty(); } }
public long countLocal(final String index, final String provider_hash) { try { SearchResponse response = elasticsearchClient.prepareSearch(index) .setSize(0) .setQuery(QueryBuilders.matchQuery("provider_hash", provider_hash)) .execute() .actionGet(); return response.getHits().getTotalHits(); } catch (Throwable e) { DAO.severe(e); return 0; } }
Vector<HashMap<String, ByteIterator>> result) { try { final RangeQueryBuilder rangeQuery = rangeQuery("_id").gte(startkey); final SearchResponse response = client.prepareSearch(indexKey) .setTypes(table) .setQuery(rangeQuery) .setSize(recordcount) .execute() .actionGet(); for (SearchHit hit : response.getHits()) { entry = new HashMap<>(fields.size()); for (String field : fields) {
/** * Get a search response for predefined aggregation task on a specific index. * @param index Name of ES index * @param aggr Pre-configured AggregationBuilder object * @return HashMap with parsed aggregations */ private SearchResponse getAggregationResponse(String index, @SuppressWarnings("rawtypes") AggregationBuilder aggr) { return this.elasticsearchClient.prepareSearch(index) .setSearchType(SearchType.QUERY_THEN_FETCH) .setQuery(QueryBuilders.matchAllQuery()) .setFrom(0) .setSize(0) .addAggregation(aggr) .execute().actionGet(); }
public LinkedHashMap<String, Long> fullDateHistogram(final String indexName, int timezoneOffset, String histogram_timefield) { // prepare request SearchRequestBuilder request = elasticsearchClient.prepareSearch(indexName) .setSearchType(SearchType.QUERY_THEN_FETCH) .setQuery(QueryBuilders.constantScoreQuery(QueryBuilders.matchAllQuery())) .setFrom(0) .setSize(0); request.clearRescorers(); request.addAggregation(AggregationBuilders.dateHistogram(histogram_timefield).field(histogram_timefield).timeZone("UTC").minDocCount(1).interval(DateHistogramInterval.DAY)); // get response SearchResponse response = request.execute().actionGet(); // evaluate date histogram: InternalHistogram<InternalHistogram.Bucket> dateCounts = response.getAggregations().get(histogram_timefield); LinkedHashMap<String, Long> list = new LinkedHashMap<>(); for (InternalHistogram.Bucket bucket : dateCounts.getBuckets()) { Calendar cal = Calendar.getInstance(DateParser.UTCtimeZone); org.joda.time.DateTime k = (org.joda.time.DateTime) bucket.getKey(); cal.setTime(k.toDate()); cal.add(Calendar.MINUTE, -timezoneOffset); long docCount = bucket.getDocCount(); list.put(DateParser.dayDateFormat.format(cal.getTime()), docCount); } return list; }
private void deleteBulkRequest(String objectId, String index, String type, String routing, String parent) { if (logger.isTraceEnabled()) { logger.trace("bulkDeleteRequest - objectId: {} - index: {} - type: {} - routing: {} - parent: {}", objectId, index, type, routing, parent); } if (definition.getParentTypes() != null && definition.getParentTypes().contains(type)) { QueryBuilder builder = QueryBuilders.hasParentQuery(type, QueryBuilders.termQuery(MongoDBRiver.MONGODB_ID_FIELD, objectId)); SearchResponse response = esClient.prepareSearch(index).setQuery(builder).setRouting(routing) .addField(MongoDBRiver.MONGODB_ID_FIELD).execute().actionGet(); for (SearchHit hit : response.getHits().getHits()) { getBulkProcessor(index, hit.getType()).deleteBulkRequest(hit.getId(), routing, objectId); } } getBulkProcessor(index, type).deleteBulkRequest(objectId, routing, parent); }