public void delete(String url) { if (url != null) { getConnection().getClient().prepareDelete(getIndex(), getType(), url.toLowerCase()).get(); } }
public void delete(String url) { if (url != null) { getConnection().getClient().prepareDelete(getIndex(), getType(), url.toLowerCase()).get(); } }
public PageableList<HttpSourceTest> filter(String prefix) { BoolQueryBuilder filter = QueryBuilders.boolQuery(); if (!Strings.isNullOrEmpty(prefix)) { filter.must(QueryBuilders .prefixQuery("url", prefix.trim())); } SearchResponse response = getConnection().getClient().prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(filter) .setSize(100) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); List<HttpSourceTest> items = Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToHttpSourceTest) .collect(Collectors.toList()); return PageableList.create(items, response.getHits().getTotalHits()); }
public PageableList<NamedQuery> filter(String prefix) { BoolQueryBuilder filter = QueryBuilders.boolQuery(); if (!Strings.isNullOrEmpty(prefix)) { filter.must(QueryBuilders .prefixQuery("name", prefix.trim())); } SearchResponse response = getConnection().getClient().prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(filter) .setSize(100) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); List<NamedQuery> items = Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToNamedQuery) .collect(Collectors.toList()); return PageableList.create(items, response.getHits().getTotalHits()); }
public List<String> suggest(String prefix) { CompletionSuggestionBuilder suggestionBuilder = new CompletionSuggestionBuilder("name_suggest") .prefix(prefix); SearchResponse response = getConnection().getClient().prepareSearch(getIndex()) .setTypes(getType()) .suggest(new SuggestBuilder().addSuggestion("suggestion", suggestionBuilder)) .setSize(100) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); return response.getSuggest().filter(CompletionSuggestion.class).stream() .flatMap(s -> s.getOptions().stream()) .sorted(Comparator.comparingDouble(Suggest.Suggestion.Entry.Option::getScore)) .map(Suggest.Suggestion.Entry.Option::getText) .map(Text::toString) .collect(Collectors.toList()); }
public List<HttpArticle> findByStatus(String status, int count) { BoolQueryBuilder filter = QueryBuilders.boolQuery() .must(QueryBuilders.termQuery(STATUS_FIELD, String.valueOf(status))); SearchResponse response = getConnection().getClient() .prepareSearch(getIndex()) .setTypes(getType()) .setSearchType(SearchType.DEFAULT) .setPostFilter(filter) .addSort(CREATED_FIELD, SortOrder.DESC) .setSize(count) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); SearchHits hits = response.getHits(); return Arrays.stream(hits.getHits()) .map(SearchHit::getSource) .map(this::mapToHttpArticle) .collect(Collectors.toList()); }
public PageableList<HttpSource> filter(String text) { BoolQueryBuilder filter = QueryBuilders.boolQuery(); if (!Strings.isNullOrEmpty(text)) { filter.must(QueryBuilders .queryStringQuery(QueryParser.escape(text.trim())) .field("search_field") .defaultOperator(QueryStringQueryBuilder.DEFAULT_OPERATOR.AND)); } SearchResponse response = getConnection().getClient().prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(filter) .setSize(100) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); List<HttpSource> items = Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToHttpSource) .sorted((o1, o2) -> o1.getName().compareToIgnoreCase(o2.getName())) .collect(Collectors.toList()); return PageableList.create(items, response.getHits().getTotalHits()); }
private static ElasticConnection getConnection(String hostname, int transportPort, String flushIntervalString, BulkProcessor.Listener listener) { System.setProperty("es.set.netty.runtime.available.processors", "false"); TimeValue flushInterval = TimeValue.parseTimeValue(flushIntervalString, TimeValue.timeValueSeconds(5), "flush"); Client client = getClient(hostname, transportPort); BulkProcessor bulkProcessor = BulkProcessor.builder(client, listener) .setFlushInterval(flushInterval) .setBulkActions(10) .setConcurrentRequests(10) .build(); return new ElasticConnection(client, bulkProcessor); }
public List<HttpSource> findEnabledSources() { BoolQueryBuilder filter = QueryBuilders.boolQuery() .must(QueryBuilders.termQuery("enabled", true)); SearchResponse response = getConnection().getClient() .prepareSearch(getIndex()) .setTypes(getType()) .setSearchType(SearchType.DEFAULT) .setPostFilter(filter) .addSort("updated", SortOrder.ASC) .setSize(10000) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); SearchHits hits = response.getHits(); return Arrays.stream(hits.getHits()) .map(SearchHit::sourceAsMap) .map(this::mapToHttpSource) .collect(Collectors.toList()); }
public void deleteAll() { Client client = getConnection().getClient(); TimeValue keepAlive = TimeValue.timeValueMinutes(10); SearchResponse response = client.prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(QueryBuilders.matchAllQuery()) .setSize(100) .setScroll(keepAlive) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); do { Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToHttpSourceTest) .map(HttpSourceTest::getUrl) .forEach(this::delete); response = client.prepareSearchScroll(response.getScrollId()) .setScroll(keepAlive) .execute() .actionGet(); } while (response.getHits().getHits().length != 0); }
public void store(HttpArticle article, Map<String, Object> fields) throws IOException { XContentBuilder jsonBuilder = jsonBuilder(); jsonBuilder.startObject(); applyFields(jsonBuilder, article, fields); jsonBuilder.endObject(); IndexRequestBuilder insert = getConnection().getClient() .prepareIndex(getIndex(), getType(), article.getUrl()) .setSource(jsonBuilder); getConnection().getProcessor().add(insert.request()); }
public void updateStatus(String url, String status) throws IOException { UpdateRequestBuilder update = getConnection().getClient() .prepareUpdate(getIndex(), getType(), url) .setDoc(jsonBuilder() .startObject() .field(STATUS_FIELD, status) .endObject()); getConnection().getProcessor().add(update.request()); }
public void deleteAll() { Client client = getConnection().getClient(); TimeValue keepAlive = TimeValue.timeValueMinutes(10); SearchResponse response = client.prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(QueryBuilders.matchAllQuery()) .setSize(100) .setScroll(keepAlive) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); do { Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToNamedQuery) .forEach(this::delete); response = client.prepareSearchScroll(response.getScrollId()) .setScroll(keepAlive) .execute() .actionGet(); } while (response.getHits().getHits().length != 0); }
public NamedQuery get(String name) { GetResponse response = getConnection().getClient() .prepareGet(getIndex(), getType(), name.toLowerCase()) .setFetchSource(true) .get(); if (response.isExists()) { return mapToNamedQuery(response.getSource()); } return null; }
public void save(HttpSourceTest hst) { try { String url = hst.getUrl().toLowerCase(); getConnection().getClient().prepareIndex(getIndex(), getType(), url) .setSource(jsonBuilder() .startObject() .field("url", url) .field("source_url", hst.getSource().trim()) .field("html", hst.getHtml() != null ? hst.getHtml().trim() : null) .field("url_accepted", hst.getUrlAccepted() != null ? hst.getUrlAccepted() : false) .field("title", hst.getTitle() != null ? hst.getTitle().trim() : null) .field("text", hst.getText() != null ? hst.getText().trim() : null) .field("date", hst.getDate() != null ? hst.getDate().trim() : null) .field("updated", new Date()) .endObject()) .get(); } catch (IOException e) { LOG.error("Failed to save HTTP source test with url '{}'", hst.getUrl()); } }
public void delete(NamedQuery nq) { if (nq != null && nq.getName() != null) { getConnection().getClient().prepareDelete(getIndex(), getType(), nq.getName()).get(); } }
public void save(NamedQuery nq) { try { getConnection().getClient().prepareIndex(getIndex(), getType(), nq.getName().toLowerCase()) .setSource(jsonBuilder() .startObject() .field("name", nq.getName()) .startObject("name_suggest") .field("input", Lists.newArrayList(nq.getName())) .endObject() .field("stemmed_case_sensitive", nq.getStemmedCaseSensitive()) .field("stemmed_case_insensitive", nq.getStemmedCaseInSensitive()) .field("not_stemmed_case_sensitive", nq.getNotStemmedCaseSensitive()) .field("not_stemmed_case_insensitive", nq.getNotStemmedCaseInSensitive()) .field("advanced", nq.getAdvanced()) .field("updated", new Date()) .endObject()) .get(); } catch (IOException e) { LOG.error("Failed to save HTTP source test with url '{}'", nq.getName()); } }
public void upsertUrlStatus(String url, String published, String source, boolean create, Enum status) throws IOException { Date now = new Date(); IndexRequestBuilder insert = getConnection().getClient() .prepareIndex(getIndex(), getType(), url) .setSource(jsonBuilder() .startObject() .field("url", url) .field("source", source) .field("created", now) .field("updated", now) .field("published", published) .field("status", String.valueOf(status)) .endObject()) .setCreate(create); UpdateRequestBuilder update = getConnection().getClient() .prepareUpdate(getIndex(), getType(), url) .setDoc(jsonBuilder() .startObject() .field("updated", now) .field("published", published) .field("status", String.valueOf(status)) .endObject()) .setUpsert(insert.request()); getConnection().getProcessor().add(create ? insert.request() : update.request()); }
public HttpSource get(String url) { GetResponse response = getConnection().getClient().prepareGet(getIndex(), getType(), url) .setFetchSource(true) .get(); if (response.isExists()) { return mapToHttpSource(response.getSource()); } return null; }
public HttpSourceTest get(String url) { GetResponse response = getConnection().getClient().prepareGet(getIndex(), getType(), url) .setFetchSource(true) .get(); if (response.isExists()) { return mapToHttpSourceTest(response.getSource()); } return null; }