public void delete(String url) { if (url != null) { getConnection().getClient().prepareDelete(getIndex(), getType(), url.toLowerCase()).get(); } }
public PageableList<HttpSourceTest> filter(String prefix) { BoolQueryBuilder filter = QueryBuilders.boolQuery(); if (!Strings.isNullOrEmpty(prefix)) { filter.must(QueryBuilders .prefixQuery("url", prefix.trim())); } SearchResponse response = getConnection().getClient().prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(filter) .setSize(100) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); List<HttpSourceTest> items = Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToHttpSourceTest) .collect(Collectors.toList()); return PageableList.create(items, response.getHits().getTotalHits()); }
public void deleteAll() { Client client = getConnection().getClient(); TimeValue keepAlive = TimeValue.timeValueMinutes(10); SearchResponse response = client.prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(QueryBuilders.matchAllQuery()) .setSize(100) .setScroll(keepAlive) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); do { Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToHttpSourceTest) .map(HttpSourceTest::getUrl) .forEach(this::delete); response = client.prepareSearchScroll(response.getScrollId()) .setScroll(keepAlive) .execute() .actionGet(); } while (response.getHits().getHits().length != 0); }
public List<HttpSourceTest> all() { BoolQueryBuilder filter = QueryBuilders.boolQuery() .must(QueryBuilders.existsQuery("source_url")); Client client = getConnection().getClient(); TimeValue keepAlive = TimeValue.timeValueMinutes(10); SearchResponse response = client.prepareSearch(getIndex()) .setTypes(getType()) .setPostFilter(filter) .setSize(100) .setScroll(keepAlive) .setFetchSource(true) .setExplain(false) .execute() .actionGet(); List<HttpSourceTest> result = Lists.newArrayList(); do { result.addAll(Arrays.stream(response.getHits().getHits()) .map(SearchHit::getSource) .filter(Objects::nonNull) .map(this::mapToHttpSourceTest) .collect(Collectors.toList())); response = client.prepareSearchScroll(response.getScrollId()) .setScroll(keepAlive) .execute() .actionGet(); } while (response.getHits().getHits().length != 0); return result; }
public void save(HttpSourceTest hst) { try { String url = hst.getUrl().toLowerCase(); getConnection().getClient().prepareIndex(getIndex(), getType(), url) .setSource(jsonBuilder() .startObject() .field("url", url) .field("source_url", hst.getSource().trim()) .field("html", hst.getHtml() != null ? hst.getHtml().trim() : null) .field("url_accepted", hst.getUrlAccepted() != null ? hst.getUrlAccepted() : false) .field("title", hst.getTitle() != null ? hst.getTitle().trim() : null) .field("text", hst.getText() != null ? hst.getText().trim() : null) .field("date", hst.getDate() != null ? hst.getDate().trim() : null) .field("updated", new Date()) .endObject()) .get(); } catch (IOException e) { LOG.error("Failed to save HTTP source test with url '{}'", hst.getUrl()); } }
public HttpSourceTest get(String url) { GetResponse response = getConnection().getClient().prepareGet(getIndex(), getType(), url) .setFetchSource(true) .get(); if (response.isExists()) { return mapToHttpSourceTest(response.getSource()); } return null; }