private void processSubAggregations(HasAggregations aggregation) { if (Filter.class.isAssignableFrom(aggregation.getClass())) { Filter filter = (Filter) aggregation; if (filter.getName().startsWith(NO_DATA_PREFIX)) { LinkedHashMap<String, Long> facet = getOrCreateFacet(filter.getName().replaceFirst(NO_DATA_PREFIX,"")); facet.put("NO_DATA", ((Filter) aggregation).getDocCount()); } } for (Aggregation sub : aggregation.getAggregations()) { processAggregation(sub); } }
private FilterAggregationBuilder buildTopFacetAggregation(String fieldName, String facetName, BoolQueryBuilder facetFilter, int size, Function<TermsAggregationBuilder, AggregationBuilder> additionalAggregationFilter) { TermsAggregationBuilder termsAggregation = buildTermsFacetAggregation(fieldName, facetName, size); AggregationBuilder improvedAggregation = additionalAggregationFilter.apply(termsAggregation); return AggregationBuilders .filter(facetName + "_filter", facetFilter) .subAggregation(improvedAggregation); }
private static AbstractAggregationBuilder createRatingFacet(String metricKey) { return AggregationBuilders.nested("nested_" + metricKey, FIELD_MEASURES) .subAggregation( AggregationBuilders.filter("filter_" + metricKey, termsQuery(FIELD_MEASURES_KEY, metricKey)) .subAggregation(filters(metricKey, new KeyedFilter("1", termQuery(FIELD_MEASURES_VALUE, 1d)), new KeyedFilter("2", termQuery(FIELD_MEASURES_VALUE, 2d)), new KeyedFilter("3", termQuery(FIELD_MEASURES_VALUE, 3d)), new KeyedFilter("4", termQuery(FIELD_MEASURES_VALUE, 4d)), new KeyedFilter("5", termQuery(FIELD_MEASURES_VALUE, 5d))))); }
.subAggregation(sum(NCLOC_KEY + "_filter_sum").field(FIELD_MEASURES_VALUE)))); .map(metric -> (Nested) response.getAggregations().get(metric)) .map(nested -> (Filter) nested.getAggregations().get(nested.getName() + "_filter")) .map(filter -> (Sum) filter.getAggregations().get(filter.getName() + "_sum")) .forEach(sum -> { String metric = sum.getName().replace("_filter_sum", "");
.path("attrValueList") .subAggregation(AggregationBuilders.filter("productAttrs") .filter(QueryBuilders.termQuery("attrValueList.type",1)) .subAggregation(AggregationBuilders.terms("attrIds") .field("attrValueList.productAttributeId") .subAggregation(AggregationBuilders.terms("attrValues")
public SearchResult(SearchResponse resp, Select select) throws SqlParseException { Aggregations aggs = resp.getAggregations(); if (aggs.get("filter") != null) { InternalFilter inf = aggs.get("filter"); aggs = inf.getAggregations(); } if (aggs.get("group by") != null) { InternalTerms terms = aggs.get("group by"); Collection<Bucket> buckets = terms.getBuckets(); this.total = buckets.size(); results = new ArrayList<>(buckets.size()); for (Bucket bucket : buckets) { Map<String, Object> aggsMap = toAggsMap(bucket.getAggregations().getAsMap()); aggsMap.put("docCount", bucket.getDocCount()); results.add(aggsMap); } } else { results = new ArrayList<>(1); this.total = 1; Map<String, Object> map = new HashMap<>(); for (Aggregation aggregation : aggs) { map.put(aggregation.getName(), covenValue(aggregation)); } results.add(map); } }
private static SecurityStandardCategoryStatistics processSecurityReportIssueSearchResults(InternalFilter categoryBucket, boolean includeCwe) { List<SecurityStandardCategoryStatistics> children = new ArrayList<>(); if (includeCwe) { ((StringTerms) categoryBucket.getAggregations().get("cwe")).getBuckets() .forEach(cweBucket -> children.add(processSecurityReportCategorySearchResults(cweBucket, cweBucket.getKeyAsString(), null))); } return processSecurityReportCategorySearchResults(categoryBucket, categoryBucket.getName(), children); }
private static void addRangeFacetIncludingNoData(SearchRequestBuilder esSearch, String metricKey, StickyFacetBuilder facetBuilder, Double... thresholds) { esSearch.addAggregation(createStickyFacet(metricKey, facetBuilder, AggregationBuilders.filter("combined_" + metricKey, matchAllQuery()) .subAggregation(createRangeFacet(metricKey, thresholds)) .subAggregation(createNoDataFacet(metricKey)))); }
private static SecurityStandardCategoryStatistics processSecurityReportCategorySearchResults(HasAggregations categoryBucket, String categoryName, @Nullable List<SecurityStandardCategoryStatistics> children) { List<StringTerms.Bucket> severityBuckets = ((StringTerms) ((InternalFilter) categoryBucket.getAggregations().get("vulnerabilities")).getAggregations().get("severity")) .getBuckets(); long vulnerabilities = severityBuckets.stream().mapToLong(b -> ((InternalValueCount) b.getAggregations().get(COUNT)).getValue()).sum(); // Worst severity having at least one issue OptionalInt severityRating = severityBuckets.stream() .filter(b -> ((InternalValueCount) b.getAggregations().get(COUNT)).getValue() != 0) .mapToInt(b -> Severity.ALL.indexOf(b.getKeyAsString()) + 1) .max(); long openSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get("openSecurityHotspots")).getAggregations().get(COUNT)) .getValue(); long toReviewSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get("toReviewSecurityHotspots")).getAggregations().get(COUNT)) .getValue(); long wontFixSecurityHotspots = ((InternalValueCount) ((InternalFilter) categoryBucket.getAggregations().get("wontFixSecurityHotspots")).getAggregations().get(COUNT)) .getValue(); return new SecurityStandardCategoryStatistics(categoryName, vulnerabilities, severityRating, toReviewSecurityHotspots, openSecurityHotspots, wontFixSecurityHotspots, children); }
private static AbstractAggregationBuilder createStickyFacet(String facetKey, StickyFacetBuilder facetBuilder, AbstractAggregationBuilder aggregationBuilder) { BoolQueryBuilder facetFilter = facetBuilder.getStickyFacetFilter(facetKey); return AggregationBuilders .global(facetKey) .subAggregation( AggregationBuilders .filter("facet_filter_" + facetKey, facetFilter) .subAggregation(aggregationBuilder)); }
List<Terms.Bucket> attrIds = ((LongTerms) ((InternalFilter)productAttrs.getProperty("productAttrs")).getAggregations().getProperty("attrIds")).getBuckets(); List<EsProductRelatedInfo.ProductAttr> attrList = new ArrayList<>(); for (Terms.Bucket attrId : attrIds) {
public FilterAggregationBuilder addSelectedItemsToFacet(String fieldName, String facetName, FilterAggregationBuilder facetTopAggregation, Function<TermsAggregationBuilder, AggregationBuilder> additionalAggregationFilter, Object... selected) { if (selected.length <= 0) { return facetTopAggregation; } String includes = Arrays.stream(selected) .filter(Objects::nonNull) .map(s -> EsUtils.escapeSpecialRegexChars(s.toString())) .collect(PIPE_JOINER); TermsAggregationBuilder selectedTerms = AggregationBuilders.terms(facetName + "_selected") .size(max(MAXIMUM_NUMBER_OF_SELECTED_ITEMS_WHOSE_DOC_COUNT_WILL_BE_CALCULATED, includes.length())) .field(fieldName) .includeExclude(new IncludeExclude(includes, null)); if (subAggregation != null) { selectedTerms = selectedTerms.subAggregation(subAggregation); } AggregationBuilder improvedAggregation = additionalAggregationFilter.apply(selectedTerms); facetTopAggregation.subAggregation(improvedAggregation); return facetTopAggregation; }
.subAggregation(AggregationBuilders.terms(AGG_TERMS) .field(field) .size(size > 0 ? size : 50) .subAggregation(AggregationBuilders.terms(AGG_TERMS) .script(new Script(ScriptType.INLINE, "painless", scriptStringBuilder.toString(), Collections.emptyMap())) .size(size > 0 ? size : 50)
private static void addStatusFacetIfNeeded(SearchOptions options, Map<String, AggregationBuilder> aggregations, StickyFacetBuilder stickyFacetBuilder) { if (options.getFacets().contains(FACET_STATUSES)) { BoolQueryBuilder facetFilter = stickyFacetBuilder.getStickyFacetFilter(FIELD_RULE_STATUS); AggregationBuilder statuses = AggregationBuilders.filter(FACET_STATUSES + "_filter", facetFilter) .subAggregation( AggregationBuilders .terms(FACET_STATUSES) .field(FIELD_RULE_STATUS) .includeExclude(new IncludeExclude(Joiner.on('|').join(ALL_STATUSES_EXCEPT_REMOVED), RuleStatus.REMOVED.toString())) .size(ALL_STATUSES_EXCEPT_REMOVED.size())); aggregations.put(FACET_STATUSES, AggregationBuilders.global(FACET_STATUSES).subAggregation(statuses)); } }
private static AbstractAggregationBuilder createRangeFacet(String metricKey, Double... thresholds) { RangeAggregationBuilder rangeAgg = AggregationBuilders.range(metricKey) .field(FIELD_MEASURES_VALUE); final int lastIndex = thresholds.length - 1; IntStream.range(0, thresholds.length) .forEach(i -> { if (i == 0) { rangeAgg.addUnboundedTo(thresholds[0]); rangeAgg.addRange(thresholds[0], thresholds[1]); } else if (i == lastIndex) { rangeAgg.addUnboundedFrom(thresholds[lastIndex]); } else { rangeAgg.addRange(thresholds[i], thresholds[i + 1]); } }); return AggregationBuilders.nested("nested_" + metricKey, FIELD_MEASURES) .subAggregation( AggregationBuilders.filter("filter_" + metricKey, termsQuery(FIELD_MEASURES_KEY, metricKey)) .subAggregation(rangeAgg)); }
private static AggregationBuilder createResolutionFacet(IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder esQuery) { String fieldName = RESOLUTIONS.getFieldName(); String facetName = RESOLUTIONS.getName(); // Same as in super.stickyFacetBuilder Map<String, QueryBuilder> resolutionFilters = Maps.newHashMap(filters); resolutionFilters.remove("__isResolved"); resolutionFilters.remove(fieldName); StickyFacetBuilder assigneeFacetBuilder = newStickyFacetBuilder(query, resolutionFilters, esQuery); BoolQueryBuilder facetFilter = assigneeFacetBuilder.getStickyFacetFilter(fieldName); FilterAggregationBuilder facetTopAggregation = assigneeFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, RESOLUTIONS.getSize()); facetTopAggregation = assigneeFacetBuilder.addSelectedItemsToFacet(fieldName, facetName, facetTopAggregation, t -> t); // Add missing facet for unresolved issues facetTopAggregation.subAggregation( addEffortAggregationIfNeeded(query, AggregationBuilders .missing(facetName + FACET_SUFFIX_MISSING) .field(fieldName))); return AggregationBuilders .global(facetName) .subAggregation(facetTopAggregation); }
private static AggregationBuilder createAssigneesFacet(IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder queryBuilder) { String fieldName = ASSIGNEES.getFieldName(); String facetName = ASSIGNEES.getName(); // Same as in super.stickyFacetBuilder Map<String, QueryBuilder> assigneeFilters = Maps.newHashMap(filters); assigneeFilters.remove(IS_ASSIGNED_FILTER); assigneeFilters.remove(fieldName); StickyFacetBuilder assigneeFacetBuilder = newStickyFacetBuilder(query, assigneeFilters, queryBuilder); BoolQueryBuilder facetFilter = assigneeFacetBuilder.getStickyFacetFilter(fieldName); FilterAggregationBuilder facetTopAggregation = assigneeFacetBuilder.buildTopFacetAggregation(fieldName, facetName, facetFilter, ASSIGNEES.getSize()); if (!query.assignees().isEmpty()) { facetTopAggregation = assigneeFacetBuilder.addSelectedItemsToFacet(fieldName, facetName, facetTopAggregation, t -> t, query.assignees().toArray()); } // Add missing facet for unassigned issues facetTopAggregation.subAggregation( addEffortAggregationIfNeeded(query, AggregationBuilders .missing(facetName + FACET_SUFFIX_MISSING) .field(fieldName))); return AggregationBuilders .global(facetName) .subAggregation(facetTopAggregation); }
RuleExtensionScope.system().getScope(), RuleExtensionScope.organization(query.getOrganization()).getScope())) .subAggregation(termsAggregation);
private void addAssignedToMeFacetIfNeeded(SearchRequestBuilder builder, SearchOptions options, IssueQuery query, Map<String, QueryBuilder> filters, QueryBuilder queryBuilder) { String uuid = userSession.getUuid(); if (!options.getFacets().contains(ASSIGNED_TO_ME.getName()) || StringUtils.isEmpty(uuid)) { return; } String fieldName = ASSIGNED_TO_ME.getFieldName(); String facetName = ASSIGNED_TO_ME.getName(); // Same as in super.stickyFacetBuilder StickyFacetBuilder assignedToMeFacetBuilder = newStickyFacetBuilder(query, filters, queryBuilder); BoolQueryBuilder facetFilter = assignedToMeFacetBuilder.getStickyFacetFilter(IS_ASSIGNED_FILTER, fieldName); FilterAggregationBuilder facetTopAggregation = AggregationBuilders .filter(facetName + "__filter", facetFilter) .subAggregation(addEffortAggregationIfNeeded(query, AggregationBuilders.terms(facetName + "__terms") .field(fieldName) .includeExclude(new IncludeExclude(escapeSpecialRegexChars(uuid), null)))); builder.addAggregation( AggregationBuilders.global(facetName) .subAggregation(facetTopAggregation)); }
private static AggregationBuilder addSecurityReportIssueCountAggregations(AggregationBuilder categoryAggs) { return categoryAggs .subAggregation( AggregationBuilders.filter("vulnerabilities", boolQuery() .filter(termQuery(FIELD_ISSUE_TYPE, RuleType.VULNERABILITY.name())) .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION))) .subAggregation( AggregationBuilders.terms("severity").field(FIELD_ISSUE_SEVERITY) .subAggregation( AggregationBuilders.count(COUNT).field(FIELD_ISSUE_KEY)))) .subAggregation(AggregationBuilders.filter("openSecurityHotspots", boolQuery() .filter(termQuery(FIELD_ISSUE_TYPE, RuleType.SECURITY_HOTSPOT.name())) .mustNot(existsQuery(FIELD_ISSUE_RESOLUTION))) .subAggregation( AggregationBuilders.count(COUNT).field(FIELD_ISSUE_KEY))) .subAggregation(AggregationBuilders.filter("toReviewSecurityHotspots", boolQuery() .filter(termQuery(FIELD_ISSUE_TYPE, RuleType.SECURITY_HOTSPOT.name())) .filter(termQuery(FIELD_ISSUE_STATUS, Issue.STATUS_RESOLVED)) .filter(termQuery(FIELD_ISSUE_RESOLUTION, Issue.RESOLUTION_FIXED))) .subAggregation( AggregationBuilders.count(COUNT).field(FIELD_ISSUE_KEY))) .subAggregation(AggregationBuilders.filter("wontFixSecurityHotspots", boolQuery() .filter(termQuery(FIELD_ISSUE_TYPE, RuleType.SECURITY_HOTSPOT.name())) .filter(termQuery(FIELD_ISSUE_STATUS, Issue.STATUS_RESOLVED)) .filter(termQuery(FIELD_ISSUE_RESOLUTION, Issue.RESOLUTION_WONT_FIX))) .subAggregation( AggregationBuilders.count(COUNT).field(FIELD_ISSUE_KEY))); }