public List<String> terms(String fields, @Nullable String query, int size) { TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME) .field(fields) .size(size) .minDocCount(1); if (query != null) { termsAggregation.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(query) + ".*", null)); } SearchRequestBuilder request = client .prepareSearch(INDEX_TYPE_RULE, INDEX_TYPE_ACTIVE_RULE) .setQuery(matchAllQuery()) .setSize(0) .addAggregation(termsAggregation); SearchResponse esResponse = request.get(); return EsUtils.termsKeys(esResponse.getAggregations().get(AGGREGATION_NAME)); }
BoolQueryBuilder esFilter = boolQuery(); request.setQuery(esFilter); request.addAggregation(AggregationBuilders.terms(FIELD_LANGUAGES) .field(FIELD_LANGUAGES) .size(MAX_PAGE_SIZE) .minDocCount(1) .order(Terms.Order.count(false))); request.addAggregation(AggregationBuilders.nested(FIELD_NCLOC_LANGUAGE_DISTRIBUTION, FIELD_NCLOC_LANGUAGE_DISTRIBUTION) .subAggregation(AggregationBuilders.terms(FIELD_NCLOC_LANGUAGE_DISTRIBUTION + "_terms") .field(FIELD_DISTRIB_LANGUAGE) .size(MAX_PAGE_SIZE) .minDocCount(1) .order(Terms.Order.count(false)) .subAggregation(sum(FIELD_DISTRIB_NCLOC).field(FIELD_DISTRIB_NCLOC)))); .subAggregation(AggregationBuilders.filter(NCLOC_KEY + "_filter", termQuery(FIELD_MEASURES_KEY, NCLOC_KEY)) statistics.setSum(metric, value); }); statistics.setProjectCountByLanguage(termsToMap(response.getAggregations().get(FIELD_LANGUAGES))); .flatMap(terms -> terms.getBuckets().stream()) .collect(MoreCollectors.uniqueIndex(Bucket::getKeyAsString, bucketToNcloc)); statistics.setNclocByLanguage(nclocByLanguage);
private TermsAggregationBuilder buildTermsFacetAggregation(String fieldName, String facetName, int size) { TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(facetName) .field(fieldName) .order(order) .size(size) .minDocCount(FACET_DEFAULT_MIN_DOC_COUNT); if (subAggregation != null) { termsAggregation = termsAggregation.subAggregation(subAggregation); } return termsAggregation; }
public List<String> searchTags(@Nullable String textQuery, int size) { int maxPageSize = 500; checkArgument(size <= maxPageSize, "Page size must be lower than or equals to " + maxPageSize); if (size <= 0) { return emptyList(); } TermsAggregationBuilder tagFacet = AggregationBuilders.terms(FIELD_TAGS) .field(FIELD_TAGS) .size(size) .minDocCount(1) .order(Terms.Order.term(true)); if (textQuery != null) { tagFacet.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(textQuery) + ".*", null)); } SearchRequestBuilder searchQuery = client .prepareSearch(INDEX_TYPE_PROJECT_MEASURES) .setQuery(authorizationTypeSupport.createQueryFilter()) .setFetchSource(false) .setSize(0) .addAggregation(tagFacet); Terms aggregation = searchQuery.get().getAggregations().get(FIELD_TAGS); return aggregation.getBuckets().stream() .map(Bucket::getKeyAsString) .collect(MoreCollectors.toList()); }
public Collection<org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket> termAggregation(String field){ SearchResponse response = client.prepareSearch(indexName) //no return for matches .setSize(0) .setQuery(QueryBuilders.matchAllQuery()) //return all terms .addAggregation(terms("agg").field(field).size(Integer.MAX_VALUE)) .execute().actionGet(); org.elasticsearch.search.aggregations.bucket.terms.Terms terms = response.getAggregations().get("agg"); Collection<org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket> buckets = terms.getBuckets(); return buckets; }
.subAggregation( AggregationBuilders.terms("branchUuid").field(FIELD_ISSUE_BRANCH_UUID) .subAggregation( AggregationBuilders.count(COUNT).field(FIELD_ISSUE_KEY)) .subAggregation( return response.getAggregations().asList().stream() .map(x -> (InternalFilter) x) .flatMap(projectBucket -> ((StringTerms) projectBucket.getAggregations().get("branchUuid")).getBuckets().stream() .flatMap(branchBucket -> { long count = ((InternalValueCount) branchBucket.getAggregations().get(COUNT)).getValue(); if (count < 1L) { return Stream.empty(); long lastIssueDate = (long) ((InternalMax) branchBucket.getAggregations().get("maxFuncCreatedAt")).getValue(); return Stream.of(new ProjectStatistics(branchBucket.getKeyAsString(), count, lastIssueDate)); }))
public List<BranchStatistics> searchBranchStatistics(String projectUuid, List<String> branchUuids) { if (branchUuids.isEmpty()) { return Collections.emptyList(); } SearchRequestBuilder request = client.prepareSearch(INDEX_TYPE_ISSUE) .setRouting(projectUuid) .setQuery( boolQuery() .must(termsQuery(FIELD_ISSUE_BRANCH_UUID, branchUuids)) .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION)) .must(termQuery(FIELD_ISSUE_IS_MAIN_BRANCH, Boolean.toString(false)))) .setSize(0) .addAggregation(AggregationBuilders.terms("branchUuids") .field(FIELD_ISSUE_BRANCH_UUID) .size(branchUuids.size()) .subAggregation(AggregationBuilders.terms("types") .field(FIELD_ISSUE_TYPE))); SearchResponse response = request.get(); return ((StringTerms) response.getAggregations().get("branchUuids")).getBuckets().stream() .map(bucket -> new BranchStatistics(bucket.getKeyAsString(), ((StringTerms) bucket.getAggregations().get("types")).getBuckets() .stream() .collect(uniqueIndex(StringTerms.Bucket::getKeyAsString, InternalTerms.Bucket::getDocCount)))) .collect(MoreCollectors.toList(branchUuids.size())); }
if (typeName != null) request.setTypes(typeName); request .setExplain(explain) .setSearchType(SearchType.QUERY_THEN_FETCH) .setQuery(queryBuilder) .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) // DFS_QUERY_THEN_FETCH is slower but provides stability of search results request.clearRescorers(); for (WebMapping field: aggregationFields) { request.addAggregation(AggregationBuilders.terms(field.getMapping().name()).field(field.getMapping().name()).minDocCount(1).size(aggregationLimit)); SearchResponse response = request.execute().actionGet(); SearchHits searchHits = response.getHits(); hitCount = (int) searchHits.getTotalHits(); Terms fieldCounts = response.getAggregations().get(field.getMapping().name()); List<? extends Bucket> buckets = fieldCounts.getBuckets();
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery() .must(QueryBuilders.matchAllQuery()) .filter(elementTypeFilterBuilder); SearchRequestBuilder q = getClient().prepareSearch(getIndexNamesAsArray(graph)) .setQuery(queryBuilder) .setSize(0); .terms(countAggName) .field(p) .size(500000); q = q.addAggregation(countAgg); SearchResponse response = checkForFailures(getClient().search(q.request()).actionGet()); Map<Object, Long> results = new HashMap<>(); for (Aggregation agg : response.getAggregations().asList()) { Terms propertyCountResults = (Terms) agg; for (Terms.Bucket propertyCountResult : propertyCountResults.getBuckets()) { String mapKey = ((String) propertyCountResult.getKey()).toLowerCase(); Long previousValue = results.get(mapKey);
SearchResponse sr = transportClient.prepareSearch(es.getIndex()).setTypes(es.getType()) //要查询的表 .setQuery(QueryBuilders.boolQuery() .must(QueryBuilders.termQuery("store_id", storeId)) //挨个设置查询条件,没有就不加,如果是字符串类型的,要加keyword后缀 .must(QueryBuilders.termQuery("store_name.keyword", storeName)) .must(QueryBuilders.rangeQuery("pay_date.keyword").gte(startDate).lte(endDate)) ).addAggregation( AggregationBuilders.terms("by_product_code").field("product_code.keyword").size(2000) //按货号分组,最多查500个货号.SKU直接改字段名字就可以 .subAggregation(AggregationBuilders.sum("quantity").field("quantity")) //分组计算销量汇总 .subAggregation(AggregationBuilders.sum("amount").field("amount")) //分组计算实付款汇总,需要加其他汇总的在这里依次加 .subAggregation(PipelineAggregatorBuilders.bucketSelector("sales_bucket_filter",script,"quantity"))//查询是否大于指定值 .order(BucketOrder.aggregation("amount", false))) //分组排序 .execute().actionGet(); Terms terms = sr.getAggregations().get("by_product_code"); //查询遍历第一个根据货号分组的aggregation System.out.println(terms.getBuckets().size()); for (Terms.Bucket entry : terms.getBuckets()) { Map<String,Object> objectMap = new HashMap<>(); System.out.println("------------------"); System.out.println("【 " + entry.getKey() + " 】订单数 : " + entry.getDocCount() ); Sum sum0 = entry.getAggregations().get("quantity"); //取得销量的汇总 Sum sum1 = entry.getAggregations().get("amount"); //取得销量的汇总
ssb.aggregation(AggregationBuilders.terms(aggregation.getName()).field(aggregation.getField())); if (response.getHits() != null) { for (SearchHit hit : response.getHits()) { ESSearchHit esSearchHit = new ESSearchHit(); if (!hit.getFields().isEmpty()) { esSearchResponse.setTotalHits(response.getHits().getTotalHits()); if (response.getAggregations() != null) { for (String name : response.getAggregations().asMap().keySet()) { Terms termsAgg = response.getAggregations().get(name); ESTermsAggregation aggregation = new ESTermsAggregation(name, null); for (Terms.Bucket bucket : termsAgg.getBuckets()) { aggregation.addBucket(new ESTermsAggregation.ESTermsBucket(bucket.getKeyAsString(), bucket.getDocCount()));
return AggregationBuilders.filter(AGG_FILTER, QueryBuilders.matchAllQuery()) .subAggregation(AggregationBuilders.terms(AGG_TERMS) .field(field) .size(size > 0 ? size : 50) .order(termsOrder)); final BoolQueryBuilder filterQuery = QueryBuilders.boolQuery(); filterQuery.must(QueryBuilders.existsQuery(field)); }); return AggregationBuilders.filter(AGG_FILTER, filterQuery) .subAggregation(AggregationBuilders.terms(AGG_TERMS) .script(new Script(ScriptType.INLINE, "painless", scriptStringBuilder.toString(), Collections.emptyMap())) .size(size > 0 ? size : 50) .order(termsOrder));
SearchResponse searchResponse = transportClient.prepareSearch("product_index").setTypes("product") .addAggregation(AggregationBuilders.terms("product_group_by_price").field("price") .subAggregation(AggregationBuilders.dateHistogram("product_group_by_created_date_time").field("created_date_time") .dateHistogramInterval(DateHistogramInterval.YEAR) .subAggregation(AggregationBuilders.avg("product_avg_price").field("price"))) ).execute().actionGet(); Map<String, Aggregation> aggregationMap = searchResponse.getAggregations().asMap(); logger.info("--------------------------------:" + productGroupByPriceBucket.getKey() + ":" + productGroupByPriceBucket.getDocCount()); Histogram productGroupByPrice1 = (Histogram) productGroupByPriceBucket.getAggregations().asMap().get("product_group_by_price"); Iterator<org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket> groupByCreateDateTimeIterator = productGroupByPrice1.getBuckets().iterator(); while (groupByCreateDateTimeIterator.hasNext()) { logger.info("--------------------------------:" + groupByCreateDateTimeBucket.getKey() + ":" + groupByCreateDateTimeBucket.getDocCount()); Avg avgPrice = (Avg) groupByCreateDateTimeBucket.getAggregations().asMap().get("product_avg_price"); logger.info("--------------------------------:" + avgPrice.getValue());
private static void addStatusFacetIfNeeded(SearchOptions options, Map<String, AggregationBuilder> aggregations, StickyFacetBuilder stickyFacetBuilder) { if (options.getFacets().contains(FACET_STATUSES)) { BoolQueryBuilder facetFilter = stickyFacetBuilder.getStickyFacetFilter(FIELD_RULE_STATUS); AggregationBuilder statuses = AggregationBuilders.filter(FACET_STATUSES + "_filter", facetFilter) .subAggregation( AggregationBuilders .terms(FACET_STATUSES) .field(FIELD_RULE_STATUS) .includeExclude(new IncludeExclude(Joiner.on('|').join(ALL_STATUSES_EXCEPT_REMOVED), RuleStatus.REMOVED.toString())) .size(ALL_STATUSES_EXCEPT_REMOVED.size())); aggregations.put(FACET_STATUSES, AggregationBuilders.global(FACET_STATUSES).subAggregation(statuses)); } }
private static AggregationBuilder addSecurityReportIssueCountAggregations(AggregationBuilder categoryAggs) { return categoryAggs .subAggregation( AggregationBuilders.filter("vulnerabilities", boolQuery() .filter(termQuery(FIELD_ISSUE_TYPE, RuleType.VULNERABILITY.name())) .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION))) .subAggregation( AggregationBuilders.terms("severity").field(FIELD_ISSUE_SEVERITY) .subAggregation( AggregationBuilders.count(COUNT).field(FIELD_ISSUE_KEY)))) .subAggregation(AggregationBuilders.filter("openSecurityHotspots", boolQuery() .filter(termQuery(FIELD_ISSUE_TYPE, RuleType.SECURITY_HOTSPOT.name())) .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION))) .subAggregation( AggregationBuilders.count(COUNT).field(FIELD_ISSUE_KEY))) .subAggregation(AggregationBuilders.filter("toReviewSecurityHotspots", boolQuery() .filter(termQuery(FIELD_ISSUE_TYPE, RuleType.SECURITY_HOTSPOT.name())) .filter(termQuery(FIELD_ISSUE_STATUS, Issue.STATUS_RESOLVED)) .filter(termQuery(FIELD_ISSUE_RESOLUTION, Issue.RESOLUTION_FIXED))) .subAggregation( AggregationBuilders.count(COUNT).field(FIELD_ISSUE_KEY))) .subAggregation(AggregationBuilders.filter("wontFixSecurityHotspots", boolQuery() .filter(termQuery(FIELD_ISSUE_TYPE, RuleType.SECURITY_HOTSPOT.name())) .filter(termQuery(FIELD_ISSUE_STATUS, Issue.STATUS_RESOLVED)) .filter(termQuery(FIELD_ISSUE_RESOLUTION, Issue.RESOLUTION_WONT_FIX))) .subAggregation( AggregationBuilders.count(COUNT).field(FIELD_ISSUE_KEY))); }
formattedQueryDate); QueryBuilder queryBuilder = QueryBuilders.rangeQuery("nextFetchDate") .lte(formattedQueryDate); queryBuilder = boolQuery().must(queryBuilder).filter( QueryBuilders.queryStringQuery(filterQuery)); .terms("partition").field(partitionField).size(maxBucketNum); TopHitsAggregationBuilder tophits = AggregationBuilders.topHits("docs") .size(maxURLsPerBucket).explain(false); aggregations.subAggregation(tophits); MinAggregationBuilder minBuilder = AggregationBuilders.min( "top_hit").field(totalSortField); aggregations.subAggregation(minBuilder); aggregations.order(BucketOrder.aggregation("top_hit", true));
public void testOtherDocCount(String... fieldNames) { for (String fieldName : fieldNames) { SearchResponse allTerms = client().prepareSearch("idx") .addAggregation(terms("terms") .executionHint(randomExecutionHint()) .field(fieldName) .size(10000) .collectMode(randomFrom(SubAggCollectionMode.values()))) .get(); assertSearchResponse(allTerms); Terms terms = allTerms.getAggregations().get("terms"); assertEquals(0, terms.getSumOfOtherDocCounts()); // size is 0 final long sumOfDocCounts = sumOfDocCounts(terms); final int totalNumTerms = terms.getBuckets().size(); .addAggregation(terms("terms") .executionHint(randomExecutionHint()) .field(fieldName) .size(size) .shardSize(shardSize) .collectMode(randomFrom(SubAggCollectionMode.values()))) .get(); assertSearchResponse(resp); terms = resp.getAggregations().get("terms"); assertEquals(Math.min(size, totalNumTerms), terms.getBuckets().size()); assertEquals(sumOfDocCounts, sumOfDocCounts(terms));
@Override public SqlElasticSearchRequestBuilder explain() throws SqlParseException { this.request = new SearchRequestBuilder(client, SearchAction.INSTANCE); //zhongshu-comment master的写法 ((TermsAggregationBuilder) lastAgg).shardSize(5000); for (Hint hint : select.getHints()) { if (hint.getType() == HintType.SHARD_SIZE) { if (hint.getParams() != null && hint.getParams().length != 0 && hint.getParams()[0] != null) { ((TermsAggregationBuilder) lastAgg).shardSize((Integer) hint.getParams()[0]); request.addAggregation(wrapNestedIfNeeded(nestedBuilder, field.isReverseNested())); } else if (field.isChildren()) { AggregationBuilder childrenBuilder = createChildrenAggregation(field); String orderName = order.getName(); if (isAliasFiled(orderName)) { termsBuilder.order(BucketOrder.aggregation(orderName, isASC(order))); } else { termsBuilder.order(BucketOrder.count(isASC(order))); termsBuilder.order(BucketOrder.key(isASC(order))); request.addSort(order.getName(), SortOrder.valueOf(order.getType())); break; case "FIELD": termsBuilder.order(BucketOrder.aggregation(order.getName(), isASC(order))); break; default:
/** * 增加属性的聚合 * * @param nativeSearchQueryBuilder 查询主体 */ private void addAttributeAggregation(NativeSearchQueryBuilder nativeSearchQueryBuilder) { nativeSearchQueryBuilder.addAggregation(AggregationBuilders.nested("esAttributes", "esAttributes").subAggregation(AggregationBuilders.terms("attributeName").field("esAttributes.attributeName").subAggregation(AggregationBuilders.terms("attributeValue").field("esAttributes.attributeValue")))); }
public FilterAggregationBuilder addSelectedItemsToFacet(String fieldName, String facetName, FilterAggregationBuilder facetTopAggregation, Function<TermsAggregationBuilder, AggregationBuilder> additionalAggregationFilter, Object... selected) { if (selected.length <= 0) { return facetTopAggregation; } String includes = Arrays.stream(selected) .filter(Objects::nonNull) .map(s -> EsUtils.escapeSpecialRegexChars(s.toString())) .collect(PIPE_JOINER); TermsAggregationBuilder selectedTerms = AggregationBuilders.terms(facetName + "_selected") .size(max(MAXIMUM_NUMBER_OF_SELECTED_ITEMS_WHOSE_DOC_COUNT_WILL_BE_CALCULATED, includes.length())) .field(fieldName) .includeExclude(new IncludeExclude(includes, null)); if (subAggregation != null) { selectedTerms = selectedTerms.subAggregation(subAggregation); } AggregationBuilder improvedAggregation = additionalAggregationFilter.apply(selectedTerms); facetTopAggregation.subAggregation(improvedAggregation); return facetTopAggregation; }