/** * Get the number of documents in the search index for a given search query * * @param q * the query * @return the count of all documents in the index which matches with the query */ private long count(final QueryBuilder q, final String indexName) { SearchResponse response = elasticsearchClient.prepareSearch(indexName).setQuery(q).setSize(0).execute().actionGet(); return response.getHits().getTotalHits(); }
@Override public void setup() throws Exception { elasticSearchClient.admin() .cluster() .prepareHealth() .setWaitForGreenStatus() .execute() .get(); try { initIndex(); updateLogIndexName(); Executors.newScheduledThreadPool(1) .scheduleAtFixedRate(() -> updateLogIndexName(), 0, 1, TimeUnit.HOURS); } catch (Exception e) { logger.error(e.getMessage(), e); } //1. Create the required index addIndex(indexName); //2. Add Mappings for the workflow document type addMappingToIndex(indexName, WORKFLOW_DOC_TYPE, "/mappings_docType_workflow.json"); //3. Add Mappings for task document type addMappingToIndex(indexName, TASK_DOC_TYPE, "/mappings_docType_task.json"); }
/** * Completely delete an index. */ public Observable deleteApplication() { String idString = applicationId(applicationScope.getApplication()); final TermQueryBuilder tqb = QueryBuilders.termQuery(APPLICATION_ID_FIELDNAME, idString); final String[] indexes = getIndexes(); //Added For Graphite Metrics return Observable.from( indexes ).flatMap( index -> { final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient().prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute(); response.addListener( new ActionListener<DeleteByQueryResponse>() { @Override public void onResponse( DeleteByQueryResponse response ) { checkDeleteByQueryResponse( tqb, response ); } @Override public void onFailure( Throwable e ) { logger.error( "Failed on delete index", e.getMessage() ); } } ); return Observable.from( response ); } ).doOnError( t -> logger.error( "Failed on delete application", t.getMessage() ) ); }
public Map<String, Object> query(final String indexName, final String fieldKey, final String fieldValue) { if (fieldKey == null || fieldValue.length() == 0) return null; // prepare request BoolQueryBuilder query = QueryBuilders.boolQuery(); query.filter(QueryBuilders.constantScoreQuery(QueryBuilders.termQuery(fieldKey, fieldValue))); SearchRequestBuilder request = elasticsearchClient.prepareSearch(indexName) .setSearchType(SearchType.QUERY_THEN_FETCH) .setQuery(query) .setFrom(0) .setSize(1) .setTerminateAfter(1); // get response SearchResponse response = request.execute().actionGet(); // evaluate search result SearchHit[] hits = response.getHits().getHits(); if (hits.length == 0) return null; assert hits.length == 1; Map<String, Object> map = hits[0].getSource(); return map; }
private void updateLogIndexName() { this.logIndexName = this.logIndexPrefix + "_" + SIMPLE_DATE_FORMAT.format(new Date()); try { elasticSearchClient.admin() .indices() .prepareGetIndex() .addIndices(logIndexName) .execute() .actionGet(); } catch (IndexNotFoundException infe) { try { elasticSearchClient.admin() .indices() .prepareCreate(logIndexName) .execute() .actionGet(); } catch (ResourceAlreadyExistsException ilee) { // no-op } catch (Exception e) { logger.error("Failed to update log index name: {}", logIndexName, e); } } }
QueryBuilder queryBuilder = expression.getFilterBuilder(); BoolQueryBuilder filterQuery = QueryBuilders.boolQuery().must(queryBuilder); QueryStringQueryBuilder stringQuery = QueryBuilders.queryStringQuery("*"); BoolQueryBuilder fq = QueryBuilders.boolQuery().must(stringQuery).must(filterQuery); final SearchRequestBuilder srb = elasticSearchClient.prepareSearch(logIndexPrefix + "*") .setQuery(fq) .setTypes(LOG_DOC_TYPE) .addSort(sortBuilder); SearchResponse response = srb.execute().actionGet(); return Arrays.stream(response.getHits().getHits()) .map(hit -> { String source = hit.getSourceAsString();
@Override public List<String> searchArchivableWorkflows(String indexName, long archiveTtlDays) { QueryBuilder q = QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("status", "COMPLETED")) .should(QueryBuilders.termQuery("status", "FAILED")) .mustNot(QueryBuilders.existsQuery("archived")) .minimumShouldMatch(1); SearchRequestBuilder s = elasticSearchClient.prepareSearch(indexName) .setTypes("workflow") .setQuery(q) .addSort("endTime", SortOrder.ASC) .setSize(archiveSearchBatchSize); SearchResponse response = s.execute().actionGet(); SearchHits hits = response.getHits(); logger.info("Archive search totalHits - {}", hits.getTotalHits()); return Arrays.stream(hits.getHits()) .map(hit -> hit.getId()) .collect(Collectors.toCollection(LinkedList::new)); }
private void deleteExtendedDataForElement(Element element) { try { QueryBuilder filter = QueryBuilders.termQuery(EXTENDED_DATA_ELEMENT_ID_FIELD_NAME, element.getId()); SearchRequestBuilder s = getClient().prepareSearch(getIndicesToQuery()) .setTypes(ELEMENT_TYPE) .setQuery(QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()).filter(filter)) .addField(EXTENDED_DATA_ELEMENT_ID_FIELD_NAME) .addField(EXTENDED_DATA_TABLE_NAME_FIELD_NAME) .addField(EXTENDED_DATA_TABLE_ROW_ID_FIELD_NAME); for (SearchHit hit : s.execute().get().getHits()) { if (MUTATION_LOGGER.isTraceEnabled()) { LOGGER.trace("deleting extended data document %s", hit.getId()); } getClient().prepareDelete(hit.getIndex(), ELEMENT_TYPE, hit.getId()).execute().actionGet(); } } catch (Exception ex) { throw new VertexiumException("Could not delete extended data for element: " + element.getId()); } }
@Override public List<String> searchRecentRunningWorkflows(int lastModifiedHoursAgoFrom, int lastModifiedHoursAgoTo) { DateTime dateTime = new DateTime(); QueryBuilder q = QueryBuilders.boolQuery() .must(QueryBuilders.rangeQuery("updateTime") .gt(dateTime.minusHours(lastModifiedHoursAgoFrom))) .must(QueryBuilders.rangeQuery("updateTime") .lt(dateTime.minusHours(lastModifiedHoursAgoTo))) .must(QueryBuilders.termQuery("status", "RUNNING")); SearchRequestBuilder s = elasticSearchClient.prepareSearch(indexName) .setTypes("workflow") .setQuery(q) .setSize(5000) .addSort("updateTime", SortOrder.ASC); SearchResponse response = s.execute().actionGet(); return StreamSupport.stream(response.getHits().spliterator(), false) .map(hit -> hit.getId()) .collect(Collectors.toCollection(LinkedList::new)); }
BoolQueryBuilder suggest = QueryBuilders.boolQuery(); if (q != null && q.length() > 0) { suggest.should(QueryBuilders.fuzzyQuery(fieldName, q).fuzziness(Fuzziness.fromEdits(2))); suggest.should(QueryBuilders.moreLikeThisQuery(fieldName).like(q)); suggest.should(QueryBuilders.matchPhrasePrefixQuery(fieldName, q)); if (q.indexOf('*') >= 0 || q.indexOf('?') >= 0) suggest.should(QueryBuilders.wildcardQuery(fieldName, q)); SearchRequestBuilder request = this.elasticsearchClient.prepareSearch(indexName) .setSearchType(SearchType.QUERY_THEN_FETCH) .setQuery(query) .setFrom(0) .setSize(resultCount) .addSort( SearchResponse response = request.execute().actionGet(); SearchHits rhits = response.getHits(); SearchHit[] hits = rhits.getHits(); for (SearchHit hit: hits) { Map<String, Object> map = hit.getSource(); result.add(map); result.setHits(rhits.getTotalHits());
public long count(final String index, final String histogram_timefield, final long millis) { try { SearchResponse response = elasticsearchClient.prepareSearch(index) .setSize(0) .setQuery(millis <= 0 ? QueryBuilders.constantScoreQuery(QueryBuilders.matchAllQuery()) : QueryBuilders.rangeQuery(histogram_timefield).from(new Date(System.currentTimeMillis() - millis))) .execute() .actionGet(); return response.getHits().getTotalHits(); } catch (Throwable e) { DAO.severe(e); return 0; } }
@Override public List<String> query(IndexQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException { SearchRequestBuilder srb = client.prepareSearch(indexName); srb.setTypes(query.getStore()); srb.setQuery(QueryBuilders.matchAllQuery()); srb.setPostFilter(getFilter(query.getCondition(),informations.get(query.getStore()))); if (!query.getOrder().isEmpty()) { List<IndexQuery.OrderEntry> orders = query.getOrder(); SearchResponse response = srb.execute().actionGet(); log.debug("Executed query [{}] in {} ms", query.getCondition(), response.getTookInMillis()); SearchHits hits = response.getHits(); if (!query.hasLimit() && hits.totalHits() >= maxResultsSize) log.warn("Query result set truncated to first [{}] elements for query: {}", maxResultsSize, query); List<String> result = new ArrayList<String>(hits.hits().length); for (SearchHit hit : hits) { result.add(hit.id());
private Query(final String indexName, QueryBuilder queryBuilder, String order_field, int resultCount) { //TODO: sort data using order_field // prepare request SearchRequestBuilder request = elasticsearchClient.prepareSearch(indexName) .setSearchType(SearchType.QUERY_THEN_FETCH) .setQuery(queryBuilder) .setFrom(0) .setSize(resultCount); request.clearRescorers(); // get response SearchResponse response = request.execute().actionGet(); hitCount = (int) response.getHits().getTotalHits(); // evaluate search result SearchHit[] hits = response.getHits().getHits(); this.result = new ArrayList<Map<String, Object>>(hitCount); for (SearchHit hit: hits) { Map<String, Object> map = hit.getSource(); this.result.add(map); } }
SearchRequestBuilder request = elasticsearchClient.prepareSearch(indexName) .setSearchType(SearchType.QUERY_THEN_FETCH) .setQuery(queryBuilder) .setFrom(0) .setSize(resultCount); request.clearRescorers(); SearchResponse response = request.execute().actionGet(); hitCount = (int) response.getHits().getTotalHits(); SearchHit[] hits = response.getHits().getHits(); this.result = new ArrayList<Map<String, Object>>(hitCount); for (SearchHit hit: hits) { for (String field: aggregationFields) { if (field.equals(histogram_timefield)) continue; // this has special handling below Terms fieldCounts = response.getAggregations().get(field); List<Bucket> buckets = fieldCounts.getBuckets();
@Override public Iterable<RawQuery.Result<String>> query(RawQuery query, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException { SearchRequestBuilder srb = client.prepareSearch(indexName); srb.setTypes(query.getStore()); srb.setQuery(QueryBuilders.queryStringQuery(query.getQuery())); srb.setFrom(query.getOffset()); if (query.hasLimit()) srb.setSize(query.getLimit()); else srb.setSize(maxResultsSize); srb.setNoFields(); //srb.setExplain(true); SearchResponse response = srb.execute().actionGet(); log.debug("Executed query [{}] in {} ms", query.getQuery(), response.getTookInMillis()); SearchHits hits = response.getHits(); if (!query.hasLimit() && hits.totalHits() >= maxResultsSize) log.warn("Query result set truncated to first [{}] elements for query: {}", maxResultsSize, query); List<RawQuery.Result<String>> result = new ArrayList<RawQuery.Result<String>>(hits.hits().length); for (SearchHit hit : hits) { result.add(new RawQuery.Result<String>(hit.id(),hit.getScore())); } return result; }
/** * Get all the indexed documents (no paginated results). Results are not sorted. */ public List<SearchHit> getDocuments(IndexType indexType) { SearchRequestBuilder req = SHARED_NODE.client().prepareSearch(indexType.getIndex()).setTypes(indexType.getType()).setQuery(matchAllQuery()); EsUtils.optimizeScrollRequest(req); req.setScroll(new TimeValue(60000)) .setSize(100); SearchResponse response = req.get(); List<SearchHit> result = newArrayList(); while (true) { Iterables.addAll(result, response.getHits()); response = SHARED_NODE.client().prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet(); // Break condition: No hits are returned if (response.getHits().getHits().length == 0) { break; } } return result; }
/** * Delete documents using a query. Check what would be deleted first with a normal search query! * Elasticsearch once provided a native prepareDeleteByQuery method, but this was removed * in later versions. Instead, there is a plugin which iterates over search results, * see https://www.elastic.co/guide/en/elasticsearch/plugins/current/plugins-delete-by-query.html * We simulate the same behaviour here without the need of that plugin. * * @param q * @return delete document count */ public int deleteByQuery(String indexName, final QueryBuilder q) { Map<String, String> ids = new TreeMap<>(); // FIXME: deprecated, "will be removed in 3.0, you should do a regular scroll instead, ordered by `_doc`" @SuppressWarnings("deprecation") SearchResponse response = elasticsearchClient.prepareSearch(indexName).setSearchType(SearchType.SCAN) .setScroll(new TimeValue(60000)).setQuery(q).setSize(100).execute().actionGet(); while (true) { // accumulate the ids here, don't delete them right now to prevent an interference of the delete with the // scroll for (SearchHit hit : response.getHits().getHits()) { ids.put(hit.getId(), hit.getType()); } response = elasticsearchClient.prepareSearchScroll(response.getScrollId()).setScroll(new TimeValue(600000)) .execute().actionGet(); // termination if (response.getHits().getHits().length == 0) break; } return deleteBulk(indexName, ids); }
Vector<HashMap<String, ByteIterator>> result) { try { final RangeQueryBuilder rangeQuery = rangeQuery("_id").gte(startkey); final SearchResponse response = client.prepareSearch(indexKey) .setTypes(table) .setQuery(rangeQuery) .setSize(recordcount) .execute() .actionGet(); for (SearchHit hit : response.getHits()) { entry = new HashMap<>(fields.size()); for (String field : fields) {
/** * Get a search response for predefined aggregation task on a specific index. * @param index Name of ES index * @param aggr Pre-configured AggregationBuilder object * @return HashMap with parsed aggregations */ private SearchResponse getAggregationResponse(String index, @SuppressWarnings("rawtypes") AggregationBuilder aggr) { return this.elasticsearchClient.prepareSearch(index) .setSearchType(SearchType.QUERY_THEN_FETCH) .setQuery(QueryBuilders.matchAllQuery()) .setFrom(0) .setSize(0) .addAggregation(aggr) .execute().actionGet(); }
Collection<Record> records = new ArrayList<Record>(); BoolQueryBuilder bqb = QueryBuilders.boolQuery(); continue; QueryStringQueryBuilder qsqb = QueryBuilders .queryString(queryString.toString().trim()) .defaultField(propName).boost(boostFactor); bqb = required ? bqb.must(qsqb) : bqb.should(qsqb); SearchResponse response = this.client.prepareSearch(this.indexName) .setTypes(this.indexType) .setSearchType(SearchType.DFS_QUERY_THEN_FETCH).setQuery(bqb) .setSize(this.maxSearchHits).execute().actionGet(); SearchHit[] results = response.getHits().getHits(); for (SearchHit hit : results) { records.add(this.readFromSource(hit.getId(), hit.getSource()));