public List<String> terms(String fields, @Nullable String query, int size) { TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME) .field(fields) .size(size) .minDocCount(1); if (query != null) { termsAggregation.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(query) + ".*", null)); } SearchRequestBuilder request = client .prepareSearch(INDEX_TYPE_RULE, INDEX_TYPE_ACTIVE_RULE) .setQuery(matchAllQuery()) .setSize(0) .addAggregation(termsAggregation); SearchResponse esResponse = request.get(); return EsUtils.termsKeys(esResponse.getAggregations().get(AGGREGATION_NAME)); }
public SearchResult(SearchResponse response, Function<Map<String, Object>, DOC> converter, TimeZone timeZone) { this.facets = new Facets(response, timeZone); this.total = response.getHits().getTotalHits(); this.docs = EsUtils.convertToDocs(response.getHits(), converter); }
/** * Return all rule ids matching the search query, without pagination nor facets */ public Iterator<Integer> searchAll(RuleQuery query) { SearchRequestBuilder esSearch = client .prepareSearch(INDEX_TYPE_RULE) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); optimizeScrollRequest(esSearch); QueryBuilder qb = buildQuery(query); Map<String, QueryBuilder> filters = buildFilters(query); BoolQueryBuilder fb = boolQuery(); for (QueryBuilder filterBuilder : filters.values()) { fb.must(filterBuilder); } esSearch.setQuery(boolQuery().must(qb).filter(fb)); SearchResponse response = esSearch.get(); return scrollIds(client, response, Integer::parseInt); }
public FilterAggregationBuilder addSelectedItemsToFacet(String fieldName, String facetName, FilterAggregationBuilder facetTopAggregation, Function<TermsAggregationBuilder, AggregationBuilder> additionalAggregationFilter, Object... selected) { if (selected.length <= 0) { return facetTopAggregation; } String includes = Arrays.stream(selected) .filter(Objects::nonNull) .map(s -> EsUtils.escapeSpecialRegexChars(s.toString())) .collect(PIPE_JOINER); TermsAggregationBuilder selectedTerms = AggregationBuilders.terms(facetName + "_selected") .size(max(MAXIMUM_NUMBER_OF_SELECTED_ITEMS_WHOSE_DOC_COUNT_WILL_BE_CALCULATED, includes.length())) .field(fieldName) .includeExclude(new IncludeExclude(includes, null)); if (subAggregation != null) { selectedTerms = selectedTerms.subAggregation(subAggregation); } AggregationBuilder improvedAggregation = additionalAggregationFilter.apply(selectedTerms); facetTopAggregation.subAggregation(improvedAggregation); return facetTopAggregation; }
public List<String> searchTags(IssueQuery query, @Nullable String textQuery, int size) { Terms terms = listTermsMatching(FIELD_ISSUE_TAGS, query, textQuery, Terms.Order.term(true), size); return EsUtils.termsKeys(terms); }
/** * Get all the indexed documents (no paginated results). Results are not sorted. */ public List<SearchHit> getDocuments(IndexType indexType) { SearchRequestBuilder req = SHARED_NODE.client().prepareSearch(indexType.getIndex()).setTypes(indexType.getType()).setQuery(matchAllQuery()); EsUtils.optimizeScrollRequest(req); req.setScroll(new TimeValue(60000)) .setSize(100); SearchResponse response = req.get(); List<SearchHit> result = newArrayList(); while (true) { Iterables.addAll(result, response.getHits()); response = SHARED_NODE.client().prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); // Break condition: No hits are returned if (response.getHits().getHits().length == 0) { break; } } return result; }
@CheckForNull public Date getNullableFieldAsDate(String key) { Object val = getNullableField(key); if (val != null) { if (val instanceof Date) { return (Date)val; } if (val instanceof Number) { return epochSecondsToDate((Number) val); } return EsUtils.parseDateTime((String) val); } return null; }
public Map<String, Long> countTags(IssueQuery query, int maxNumberOfTags) { Terms terms = listTermsMatching(FIELD_ISSUE_TAGS, query, null, Terms.Order.count(false), maxNumberOfTags); return EsUtils.termsToMap(terms); }
@Test public void getFieldAsDate() { BaseDoc doc = new BaseDoc(Maps.newHashMap()) { @Override public String getId() { return null; } @Override public String getRouting() { return null; } @Override public String getParent() { return null; } }; Date now = new Date(); doc.setField("javaDate", now); assertThat(doc.getFieldAsDate("javaDate")).isEqualToIgnoringMillis(now); doc.setField("stringDate", EsUtils.formatDateTime(now)); assertThat(doc.getFieldAsDate("stringDate")).isEqualToIgnoringMillis(now); }
@Test public void test_escapeSpecialRegexChars() { assertThat(escapeSpecialRegexChars("")).isEqualTo(""); assertThat(escapeSpecialRegexChars("foo")).isEqualTo("foo"); assertThat(escapeSpecialRegexChars("FOO")).isEqualTo("FOO"); assertThat(escapeSpecialRegexChars("foo++")).isEqualTo("foo\\+\\+"); assertThat(escapeSpecialRegexChars("foo[]")).isEqualTo("foo\\[\\]"); assertThat(escapeSpecialRegexChars(".*")).isEqualTo("\\.\\*"); assertThat(escapeSpecialRegexChars("foo\\d")).isEqualTo("foo\\\\d"); assertThat(escapeSpecialRegexChars("^")).isEqualTo("\\^"); assertThat(escapeSpecialRegexChars("$")).isEqualTo("\\$"); assertThat(escapeSpecialRegexChars("|")).isEqualTo("\\|"); assertThat(escapeSpecialRegexChars("<")).isEqualTo("\\<"); assertThat(escapeSpecialRegexChars(">")).isEqualTo("\\>"); assertThat(escapeSpecialRegexChars("\"")).isEqualTo("\\\""); assertThat(escapeSpecialRegexChars("#")).isEqualTo("\\#"); assertThat(escapeSpecialRegexChars("~")).isEqualTo("\\~"); assertThat(escapeSpecialRegexChars("$")).isEqualTo("\\$"); assertThat(escapeSpecialRegexChars("&")).isEqualTo("\\&"); assertThat(escapeSpecialRegexChars("?")).isEqualTo("\\?"); assertThat(escapeSpecialRegexChars("a bit of | & #<\"$ .* ^ everything")).isEqualTo("a bit of \\| \\& \\#\\<\\\"\\$ \\.\\* \\^ everything"); } }
public List<String> searchAuthors(IssueQuery query, @Nullable String textQuery, int maxNumberOfAuthors) { Terms terms = listTermsMatching(FIELD_ISSUE_AUTHOR_LOGIN, query, textQuery, Terms.Order.term(true), maxNumberOfAuthors); return EsUtils.termsKeys(terms); }
public Date getFieldAsDate(String key) { Object value = getField(key); if (value instanceof Date) { return (Date)value; } if (value instanceof Number) { return epochSecondsToDate((Number) value); } return EsUtils.parseDateTime((String)value); }
statistics.setSum(metric, value); }); statistics.setProjectCountByLanguage(termsToMap(response.getAggregations().get(FIELD_LANGUAGES))); Function<Terms.Bucket, Long> bucketToNcloc = bucket -> Math.round(((Sum) bucket.getAggregations().get(FIELD_DISTRIB_NCLOC)).getValue()); Map<String, Long> nclocByLanguage = Stream.of((Nested) response.getAggregations().get(FIELD_NCLOC_LANGUAGE_DISTRIBUTION))
@Test public void getNullableFieldAsDate() { BaseDoc doc = new BaseDoc(Maps.newHashMap()) { @Override public String getId() { return null; } @Override public String getRouting() { return null; } @Override public String getParent() { return null; } }; Date now = new Date(); doc.setField("javaDate", now); assertThat(doc.getNullableFieldAsDate("javaDate")).isEqualToIgnoringMillis(now); doc.setField("stringDate", EsUtils.formatDateTime(now)); assertThat(doc.getNullableFieldAsDate("stringDate")).isEqualToIgnoringMillis(now); doc.setField("noValue", null); assertThat(doc.getNullableFieldAsDate("noValue")).isNull(); } }
public List<String> terms(String fields, @Nullable String query, int size) { TermsAggregationBuilder termsAggregation = AggregationBuilders.terms(AGGREGATION_NAME) .field(fields) .size(size) .minDocCount(1); if (query != null) { termsAggregation.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(query) + ".*", null)); } SearchRequestBuilder request = client .prepareSearch(INDEX_TYPE_RULE, INDEX_TYPE_ACTIVE_RULE) .setQuery(matchAllQuery()) .setSize(0) .addAggregation(termsAggregation); SearchResponse esResponse = request.get(); return EsUtils.termsKeys(esResponse.getAggregations().get(AGGREGATION_NAME)); }
public List<String> searchTags(@Nullable String textQuery, int size) { int maxPageSize = 500; checkArgument(size <= maxPageSize, "Page size must be lower than or equals to " + maxPageSize); if (size <= 0) { return emptyList(); } TermsAggregationBuilder tagFacet = AggregationBuilders.terms(FIELD_TAGS) .field(FIELD_TAGS) .size(size) .minDocCount(1) .order(Terms.Order.term(true)); if (textQuery != null) { tagFacet.includeExclude(new IncludeExclude(".*" + escapeSpecialRegexChars(textQuery) + ".*", null)); } SearchRequestBuilder searchQuery = client .prepareSearch(INDEX_TYPE_PROJECT_MEASURES) .setQuery(authorizationTypeSupport.createQueryFilter()) .setFetchSource(false) .setSize(0) .addAggregation(tagFacet); Terms aggregation = searchQuery.get().getAggregations().get(FIELD_TAGS); return aggregation.getBuckets().stream() .map(Bucket::getKeyAsString) .collect(MoreCollectors.toList()); }
/** * Return all rule ids matching the search query, without pagination nor facets */ public Iterator<Integer> searchAll(RuleQuery query) { SearchRequestBuilder esSearch = client .prepareSearch(INDEX_TYPE_RULE) .setScroll(TimeValue.timeValueMinutes(SCROLL_TIME_IN_MINUTES)); optimizeScrollRequest(esSearch); QueryBuilder qb = buildQuery(query); Map<String, QueryBuilder> filters = buildFilters(query); BoolQueryBuilder fb = boolQuery(); for (QueryBuilder filterBuilder : filters.values()) { fb.must(filterBuilder); } esSearch.setQuery(boolQuery().must(qb).filter(fb)); SearchResponse response = esSearch.get(); return scrollIds(client, response, Integer::parseInt); }
return EsUtils.termsKeys(esResponse.getAggregations().get(AGGREGATION_NAME_FOR_TAGS));
@Test public void convertToDocs_empty() { SearchHits hits = mock(SearchHits.class, Mockito.RETURNS_MOCKS); List<BaseDoc> docs = EsUtils.convertToDocs(hits, IssueDoc::new); assertThat(docs).isEmpty(); }
@CheckForNull public Date getNullableFieldAsDate(String key) { Object val = getNullableField(key); if (val != null) { if (val instanceof Date) { return (Date)val; } if (val instanceof Number) { return epochSecondsToDate((Number) val); } return EsUtils.parseDateTime((String) val); } return null; }