How to use

ElasticConnection

in

lt.tokenmill.crawling.es

public static ElasticConnection getConnection(String hostname, int transportPort) {
  return getConnection(hostname, transportPort, DEFAULT_FLUSH_INTERVAL_STRING, DEFAULT_BULK_LISTENER);
}

private static ElasticConnection getConnection(String hostname, int transportPort, String flushIntervalString, BulkProcessor.Listener listener) {
  System.setProperty("es.set.netty.runtime.available.processors", "false");
  TimeValue flushInterval = TimeValue.parseTimeValue(flushIntervalString, TimeValue.timeValueSeconds(5), "flush");
  Client client = getClient(hostname, transportPort);
  BulkProcessor bulkProcessor = BulkProcessor.builder(client, listener)
      .setFlushInterval(flushInterval)
      .setBulkActions(10)
      .setConcurrentRequests(10)
      .build();
  return new ElasticConnection(client, bulkProcessor);
}

  public void close() {
    if (connection != null) {
      connection.close();
    }
  }
}

public void delete(String url) {
  if (url != null) {
    getConnection().getClient().prepareDelete(getIndex(), getType(), url.toLowerCase()).get();
  }
}

public void updateStatus(String url, String status) throws IOException {
  UpdateRequestBuilder update = getConnection().getClient()
      .prepareUpdate(getIndex(), getType(), url)
      .setDoc(jsonBuilder()
          .startObject()
          .field(STATUS_FIELD, status)
          .endObject());
  getConnection().getProcessor().add(update.request());
}

public void delete(String url) {
  if (url != null) {
    getConnection().getClient().prepareDelete(getIndex(), getType(), url.toLowerCase()).get();
  }
}

public void store(HttpArticle article, Map<String, Object> fields) throws IOException {
  XContentBuilder jsonBuilder = jsonBuilder();
  jsonBuilder.startObject();
  applyFields(jsonBuilder, article, fields);
  jsonBuilder.endObject();
  IndexRequestBuilder insert = getConnection().getClient()
      .prepareIndex(getIndex(), getType(), article.getUrl())
      .setSource(jsonBuilder);
  getConnection().getProcessor().add(insert.request());
}

public PageableList<HttpSourceTest> filter(String prefix) {
  BoolQueryBuilder filter = QueryBuilders.boolQuery();
  if (!Strings.isNullOrEmpty(prefix)) {
    filter.must(QueryBuilders
        .prefixQuery("url", prefix.trim()));
  }
  SearchResponse response = getConnection().getClient().prepareSearch(getIndex())
      .setTypes(getType())
      .setPostFilter(filter)
      .setSize(100)
      .setFetchSource(true)
      .setExplain(false)
      .execute()
      .actionGet();
  List<HttpSourceTest> items = Arrays.stream(response.getHits().getHits())
      .map(SearchHit::getSource)
      .filter(Objects::nonNull)
      .map(this::mapToHttpSourceTest)
      .collect(Collectors.toList());
  return PageableList.create(items, response.getHits().getTotalHits());
}

public void upsertUrlStatus(String url, String published, String source, boolean create, Enum status) throws IOException {
  Date now = new Date();
  IndexRequestBuilder insert = getConnection().getClient()
      .prepareIndex(getIndex(), getType(), url)
      .setSource(jsonBuilder()
          .startObject()
          .field("url", url)
          .field("source", source)
          .field("created", now)
          .field("updated", now)
          .field("published", published)
          .field("status", String.valueOf(status))
          .endObject())
      .setCreate(create);
  UpdateRequestBuilder update = getConnection().getClient()
      .prepareUpdate(getIndex(), getType(), url)
      .setDoc(jsonBuilder()
          .startObject()
          .field("updated", now)
          .field("published", published)
          .field("status", String.valueOf(status))
          .endObject())
      .setUpsert(insert.request());
  getConnection().getProcessor().add(create ? insert.request() : update.request());
}

public static ElasticConnection getConnection(String hostname) {
  return getConnection(hostname, DEFAULT_TRANSPORT_PORT, DEFAULT_FLUSH_INTERVAL_STRING, DEFAULT_BULK_LISTENER);
}

public PageableList<NamedQuery> filter(String prefix) {
  BoolQueryBuilder filter = QueryBuilders.boolQuery();
  if (!Strings.isNullOrEmpty(prefix)) {
    filter.must(QueryBuilders
        .prefixQuery("name", prefix.trim()));
  }
  SearchResponse response = getConnection().getClient().prepareSearch(getIndex())
      .setTypes(getType())
      .setPostFilter(filter)
      .setSize(100)
      .setFetchSource(true)
      .setExplain(false)
      .execute()
      .actionGet();
  List<NamedQuery> items = Arrays.stream(response.getHits().getHits())
      .map(SearchHit::getSource)
      .filter(Objects::nonNull)
      .map(this::mapToNamedQuery)
      .collect(Collectors.toList());
  return PageableList.create(items, response.getHits().getTotalHits());
}

  private static ElasticConnection getEsConnection() {
    if (CONNECTION == null) {
      String hostname = Configuration.INSTANCE.getString(ElasticConstants.ES_HOSTNAME_PARAM, "localhost");
      int transportPort = Configuration.INSTANCE.getInt(ElasticConstants.ES_TRANSPORT_PORT_PARAM, 9300);
      CONNECTION = ElasticConnection.getConnection(hostname, transportPort);
    }
    return CONNECTION;
  }
}

public List<String> suggest(String prefix) {
  CompletionSuggestionBuilder suggestionBuilder = new CompletionSuggestionBuilder("name_suggest")
      .prefix(prefix);
  SearchResponse response = getConnection().getClient().prepareSearch(getIndex())
      .setTypes(getType())
      .suggest(new SuggestBuilder().addSuggestion("suggestion", suggestionBuilder))
      .setSize(100)
      .setFetchSource(true)
      .setExplain(false)
      .execute()
      .actionGet();
  return response.getSuggest().filter(CompletionSuggestion.class).stream()
      .flatMap(s -> s.getOptions().stream())
      .sorted(Comparator.comparingDouble(Suggest.Suggestion.Entry.Option::getScore))
      .map(Suggest.Suggestion.Entry.Option::getText)
      .map(Text::toString)
      .collect(Collectors.toList());
}

public List<HttpArticle> findByStatus(String status, int count) {
  BoolQueryBuilder filter = QueryBuilders.boolQuery()
      .must(QueryBuilders.termQuery(STATUS_FIELD, String.valueOf(status)));
  SearchResponse response = getConnection().getClient()
      .prepareSearch(getIndex())
      .setTypes(getType())
      .setSearchType(SearchType.DEFAULT)
      .setPostFilter(filter)
      .addSort(CREATED_FIELD, SortOrder.DESC)
      .setSize(count)
      .setFetchSource(true)
      .setExplain(false)
      .execute()
      .actionGet();
  SearchHits hits = response.getHits();
  return Arrays.stream(hits.getHits())
      .map(SearchHit::getSource)
      .map(this::mapToHttpArticle)
      .collect(Collectors.toList());
}

public List<NamedQuery> all() {
  Client client = getConnection().getClient();
  TimeValue keepAlive = TimeValue.timeValueMinutes(10);
  SearchResponse response = client.prepareSearch(getIndex())
      .setTypes(getType())
      .setQuery(QueryBuilders.matchAllQuery())
      .setSize(100)
      .setScroll(keepAlive)
      .setFetchSource(true)
      .setExplain(false)
      .execute()
      .actionGet();
  List<NamedQuery> result = Lists.newArrayList();
  do {
    result.addAll(Arrays.stream(response.getHits().getHits())
        .map(SearchHit::getSource)
        .filter(Objects::nonNull)
        .map(this::mapToNamedQuery)
        .collect(Collectors.toList()));
    response = client.prepareSearchScroll(response.getScrollId())
        .setScroll(keepAlive)
        .execute()
        .actionGet();
  } while (response.getHits().getHits().length != 0);
  return result;
}

public PageableList<HttpSource> filter(String text) {
  BoolQueryBuilder filter = QueryBuilders.boolQuery();
  if (!Strings.isNullOrEmpty(text)) {
    filter.must(QueryBuilders
        .queryStringQuery(QueryParser.escape(text.trim()))
        .field("search_field")
        .defaultOperator(QueryStringQueryBuilder.DEFAULT_OPERATOR.AND));
  }
  SearchResponse response = getConnection().getClient().prepareSearch(getIndex())
      .setTypes(getType())
      .setPostFilter(filter)
      .setSize(100)
      .setFetchSource(true)
      .setExplain(false)
      .execute()
      .actionGet();
  List<HttpSource> items = Arrays.stream(response.getHits().getHits())
      .map(SearchHit::getSource)
      .filter(Objects::nonNull)
      .map(this::mapToHttpSource)
      .sorted((o1, o2) -> o1.getName().compareToIgnoreCase(o2.getName()))
      .collect(Collectors.toList());
  return PageableList.create(items, response.getHits().getTotalHits());
}

public List<HttpSource> findEnabledSources() {
  BoolQueryBuilder filter = QueryBuilders.boolQuery()
      .must(QueryBuilders.termQuery("enabled", true));
  SearchResponse response = getConnection().getClient()
      .prepareSearch(getIndex())
      .setTypes(getType())
      .setSearchType(SearchType.DEFAULT)
      .setPostFilter(filter)
      .addSort("updated", SortOrder.ASC)
      .setSize(10000)
      .setFetchSource(true)
      .setExplain(false)
      .execute()
      .actionGet();
  SearchHits hits = response.getHits();
  return Arrays.stream(hits.getHits())
      .map(SearchHit::sourceAsMap)
      .map(this::mapToHttpSource)
      .collect(Collectors.toList());
}

public void deleteAll() {
  Client client = getConnection().getClient();
  TimeValue keepAlive = TimeValue.timeValueMinutes(10);
  SearchResponse response = client.prepareSearch(getIndex())
      .setTypes(getType())
      .setPostFilter(QueryBuilders.matchAllQuery())
      .setSize(100)
      .setScroll(keepAlive)
      .setFetchSource(true)
      .setExplain(false)
      .execute()
      .actionGet();
  do {
    Arrays.stream(response.getHits().getHits())
        .map(SearchHit::getSource)
        .filter(Objects::nonNull)
        .map(this::mapToHttpSource)
        .map(HttpSource::getUrl)
        .forEach(this::delete);
    response = client.prepareSearchScroll(response.getScrollId())
        .setScroll(keepAlive)
        .execute()
        .actionGet();
  } while (response.getHits().getHits().length != 0);
}

public void deleteAll() {
  Client client = getConnection().getClient();
  TimeValue keepAlive = TimeValue.timeValueMinutes(10);
  SearchResponse response = client.prepareSearch(getIndex())
      .setTypes(getType())
      .setPostFilter(QueryBuilders.matchAllQuery())
      .setSize(100)
      .setScroll(keepAlive)
      .setFetchSource(true)
      .setExplain(false)
      .execute()
      .actionGet();
  do {
    Arrays.stream(response.getHits().getHits())
        .map(SearchHit::getSource)
        .filter(Objects::nonNull)
        .map(this::mapToHttpSourceTest)
        .map(HttpSourceTest::getUrl)
        .forEach(this::delete);
    response = client.prepareSearchScroll(response.getScrollId())
        .setScroll(keepAlive)
        .execute()
        .actionGet();
  } while (response.getHits().getHits().length != 0);
}

public List<HttpSourceTest> all() {
  BoolQueryBuilder filter = QueryBuilders.boolQuery()
      .must(QueryBuilders.existsQuery("source_url"));
  Client client = getConnection().getClient();
  TimeValue keepAlive = TimeValue.timeValueMinutes(10);
  SearchResponse response = client.prepareSearch(getIndex())
      .setTypes(getType())
      .setPostFilter(filter)
      .setSize(100)
      .setScroll(keepAlive)
      .setFetchSource(true)
      .setExplain(false)
      .execute()
      .actionGet();
  List<HttpSourceTest> result = Lists.newArrayList();
  do {
    result.addAll(Arrays.stream(response.getHits().getHits())
        .map(SearchHit::getSource)
        .filter(Objects::nonNull)
        .map(this::mapToHttpSourceTest)
        .collect(Collectors.toList()));
    response = client.prepareSearchScroll(response.getScrollId())
        .setScroll(keepAlive)
        .execute()
        .actionGet();
  } while (response.getHits().getHits().length != 0);
  return result;
}