private void logStatistics(long duration) { if (definition.isStoreStatistics()) { long totalDocuments = deletedDocuments.get() + insertedDocuments.get(); logger.trace("Indexed {} documents: {} insertions, {} updates, {} deletions", totalDocuments, insertedDocuments.get(), updatedDocuments.get(), deletedDocuments.get()); Map<String, Object> source = new HashMap<String, Object>(); Map<String, Object> statistics = Maps.newHashMap(); statistics.put("duration", duration); statistics.put("date", new Date()); statistics.put("index", index); statistics.put("type", type); statistics.put("documents.inserted", insertedDocuments.get()); statistics.put("documents.updated", updatedDocuments.get()); statistics.put("documents.deleted", deletedDocuments.get()); statistics.put("documents.total", documentCount.get()); source.put("statistics", statistics); client.prepareIndex(definition.getStatisticsIndexName(), definition.getStatisticsTypeName()).setSource(source).get(); } } }
public static void setRiverStatus(Client client, String riverName, Status status) { logger.info("setRiverStatus called with {} - {}", riverName, status); XContentBuilder xb; try { xb = jsonBuilder().startObject().startObject(MongoDBRiver.TYPE).field(MongoDBRiver.STATUS_FIELD, status).endObject() .endObject(); client.prepareIndex("_river", riverName, MongoDBRiver.STATUS_ID).setSource(xb).get(); } catch (IOException ioEx) { logger.error("setRiverStatus failed for river {}", ioEx, riverName); } }
@Override public Status insert(String table, String key, Map<String, ByteIterator> values) { try (XContentBuilder doc = jsonBuilder()) { doc.startObject(); for (final Entry<String, String> entry : StringByteIterator.getStringMap(values).entrySet()) { doc.field(entry.getKey(), entry.getValue()); } doc.field(KEY, key); doc.endObject(); final IndexResponse indexResponse = client.prepareIndex(indexKey, table).setSource(doc).get(); if (indexResponse.getResult() != DocWriteResponse.Result.CREATED) { return Status.ERROR; } if (!isRefreshNeeded) { synchronized (this) { isRefreshNeeded = true; } } return Status.OK; } catch (final Exception e) { e.printStackTrace(); return Status.ERROR; } }
private void setMetadata(String id, String value) { esClient.prepareIndex(MetadataIndexDefinition.INDEX_TYPE_METADATA) .setId(id) .setSource(MetadataIndexDefinition.FIELD_VALUE, value) .setRefreshPolicy(REFRESH_IMMEDIATE) .get(); } }
@Override public Status update(final String table, final String key, final Map<String, ByteIterator> values) { try { final SearchResponse response = search(table, key); if (response.getHits().totalHits == 0) { return Status.NOT_FOUND; } final SearchHit hit = response.getHits().getAt(0); for (final Entry<String, String> entry : StringByteIterator.getStringMap(values).entrySet()) { hit.getSource().put(entry.getKey(), entry.getValue()); } final IndexResponse indexResponse = client.prepareIndex(indexKey, table, hit.getId()).setSource(hit.getSource()).get(); if (indexResponse.getResult() != DocWriteResponse.Result.UPDATED) { return Status.ERROR; } if (!isRefreshNeeded) { synchronized (this) { isRefreshNeeded = true; } } return Status.OK; } catch (final Exception e) { e.printStackTrace(); return Status.ERROR; } }
@Test public void fail_if_bad_query() { IndexRequestBuilder requestBuilder = es.client().prepareIndex(new IndexType("unknownIndex", "unknownType")); try { requestBuilder.get(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class); assertThat(e.getMessage()).contains("Fail to execute ES index request for key 'null' on index 'unknownIndex' on type 'unknownType'"); } }
@Test public void fail_if_bad_query_with_basic_profiling() { IndexRequestBuilder requestBuilder = es.client().prepareIndex(new IndexType("unknownIndex", "unknownType")); try { requestBuilder.get(); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class); assertThat(e.getMessage()).contains("Fail to execute ES index request for key 'null' on index 'unknownIndex' on type 'unknownType'"); } }
@Test public void get_with_string_timeout_is_not_yet_implemented() { try { es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE).get("1"); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); } }
@Test public void get_with_time_value_timeout_is_not_yet_implemented() { try { es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE).get(TimeValue.timeValueMinutes(1)); fail(); } catch (Exception e) { assertThat(e).isInstanceOf(IllegalStateException.class).hasMessage("Not yet implemented"); } }
private void addToIndex(String projectUuid, String name) { esClient.prepareIndex(INDEX_TYPE_FOO) .setRouting(projectUuid) .setParent(projectUuid) .setSource(ImmutableMap.of( FooIndexDefinition.FIELD_NAME, name, FooIndexDefinition.FIELD_PROJECT_UUID, projectUuid)) .get(); }
@Override public void addResults(QueryStatusInfo statusInfo, QueryData data) { if (types.get() == null && statusInfo.getColumns() != null) { types.set(getTypes(statusInfo.getColumns())); } if (data.getData() == null) { return; } checkState(types.get() != null, "Type information is missing"); List<Column> columns = statusInfo.getColumns(); for (List<Object> fields : data.getData()) { try { XContentBuilder dataBuilder = jsonBuilder().startObject(); for (int i = 0; i < fields.size(); i++) { Type type = types.get().get(i); Object value = convertValue(fields.get(i), type); dataBuilder.field(columns.get(i).getName(), value); } dataBuilder.endObject(); client.prepareIndex(tableName, "doc") .setSource(dataBuilder.string(), JSON) .get(); } catch (IOException e) { throw new UncheckedIOException("Error loading data into Elasticsearch index: " + tableName, e); } } }
/** * Tests writing a document to a new index to ensure it's working correctly. See this post: * http://s.apache.org/index-missing-exception */ private void testNewIndex() { // create the document, this ensures the index is ready // Immediately create a document and remove it to ensure the entire cluster is ready // to receive documents. Occasionally we see errors. // See this post: http://s.apache.org/index-missing-exception if (logger.isTraceEnabled()) { logger.trace("Testing new index name: read {} write {}", alias.getReadAlias(), alias.getWriteAlias()); } final RetryOperation retryOperation = () -> { final String tempId = UUIDGenerator.newTimeUUID().toString(); esProvider.getClient().prepareIndex( alias.getWriteAlias(), VERIFY_TYPE, tempId ) .setSource(DEFAULT_PAYLOAD).get(); if (logger.isTraceEnabled()) { logger.trace("Successfully created new document with docId {} in index read {} write {} and type {}", tempId, alias.getReadAlias(), alias.getWriteAlias(), VERIFY_TYPE); } // delete all types, this way if we miss one it will get cleaned up esProvider.getClient().prepareDelete( alias.getWriteAlias(), VERIFY_TYPE, tempId).get(); if (logger.isTraceEnabled()) { logger.trace("Successfully deleted documents in read {} write {} and type {} with id {}", alias.getReadAlias(), alias.getWriteAlias(), VERIFY_TYPE, tempId); } return true; }; doInRetry(retryOperation); }
@Test public void trace_logs() { logTester.setLevel(LoggerLevel.TRACE); IndexResponse response = es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE) .setSource(FakeIndexDefinition.newDoc(42).getFields()) .get(); assertThat(response.getResult()).isSameAs(Result.CREATED); assertThat(logTester.logs(LoggerLevel.TRACE)).hasSize(1); }
@Test public void recreate_index_on_definition_changes() { // v1 startNewCreator(new FakeIndexDefinition()); IndexType fakeIndexType = new IndexType("fakes", "fake"); String id = "1"; es.client().prepareIndex(fakeIndexType).setId(id).setSource(new FakeDoc().getFields()).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue(); // v2 startNewCreator(new FakeIndexDefinitionV2()); ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = mappings(); MappingMetaData mapping = mappings.get("fakes").get("fake"); assertThat(countMappingFields(mapping)).isEqualTo(3); assertThat(field(mapping, "updatedAt").get("type")).isEqualTo("date"); assertThat(field(mapping, "newField").get("type")).isEqualTo("integer"); assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isFalse(); }
@Test public void index_with_index_type_and_id() { IndexResponse response = es.client().prepareIndex(FakeIndexDefinition.INDEX_TYPE_FAKE) .setSource(FakeIndexDefinition.newDoc(42).getFields()) .get(); assertThat(response.getResult()).isSameAs(Result.CREATED); }
@Test public void do_not_recreate_index_on_unchanged_definition() { // v1 startNewCreator(new FakeIndexDefinition()); IndexType fakeIndexType = new IndexType("fakes", "fake"); String id = "1"; es.client().prepareIndex(fakeIndexType).setId(id).setSource(new FakeDoc().getFields()).setRefreshPolicy(IMMEDIATE).get(); assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue(); // v1 startNewCreator(new FakeIndexDefinition()); assertThat(es.client().prepareGet(fakeIndexType, id).get().isExists()).isTrue(); }
public void index(TodoView todoView) { try { IndexResponse ir = transportClient .prepareIndex(TodoView.INDEX, TodoView.TYPE, todoView.getId()) .setSource(objectMapper.writeValueAsString(todoView), XContentType.JSON) .get(); } catch (JsonProcessingException e) { throw new RuntimeException(e); } }
/** * 向ES中添加数据 * @param index 索引 * @param type 类型 * @param param 要添加的数据,Map类型 * @return */ public boolean insert(String index,String type,Map<String,Object> param){ return client.prepareIndex(index,type).setSource(param).get().status().name() .equals(ESRestStatus.CREATED.name()); }
private void setMetadata(String id, String value) { esClient.prepareIndex(MetadataIndexDefinition.INDEX_TYPE_METADATA) .setId(id) .setSource(MetadataIndexDefinition.FIELD_VALUE, value) .setRefreshPolicy(REFRESH_IMMEDIATE) .get(); } }
private void createStoredQuery(String instanceId, String ttl, BooleanAgentDocument booleanAgentDocument, Optional<String> booleanRestriction) throws IOException { String reference = booleanAgentDocument.getReference() + "_" + instanceId; XContentBuilder storedQuery = prepareStoredQuery(instanceId, booleanAgentDocument, booleanRestriction, reference); // replaces an existing document with a new document by using the reference value as the Elasticsearch _id. getElasticClient() .prepareIndex(policyIndexName, percolatorTypeName, reference) .setSource(storedQuery) .setRefresh(true) .setTTL(ttl) .get(elasticsearchProperties.getElasticsearchSearchTimeout()); }