@Override public void process(final InputStream in) throws IOException { String json = IOUtils.toString(in, charset) .replace("\r\n", " ").replace('\n', ' ').replace('\r', ' '); if (indexOp.equalsIgnoreCase("index")) { bulk.add(esClient.get().prepareIndex(index, docType, id) .setSource(json.getBytes(charset))); } else if (indexOp.equalsIgnoreCase("upsert")) { bulk.add(esClient.get().prepareUpdate(index, docType, id) .setDoc(json.getBytes(charset)) .setDocAsUpsert(true)); } else if (indexOp.equalsIgnoreCase("update")) { bulk.add(esClient.get().prepareUpdate(index, docType, id) .setDoc(json.getBytes(charset))); } else { throw new IOException("Index operation: " + indexOp + " not supported."); } } });
} else { String script = getDeletionScript(informations, storename, mutation); brb.add(client.prepareUpdate(indexName, storename, docid).setScript(script, ScriptService.ScriptType.INLINE)); log.trace("Adding script {}", script); UpdateRequestBuilder update = client.prepareUpdate(indexName, storename, docid).setScript(script, ScriptService.ScriptType.INLINE); if (needUpsert) { XContentBuilder doc = getNewDocument(mutation.getAdditions(), informations.get(storename), ttl);
public void update(PhotonDoc doc) { try { this.bulkRequest.add(this.esClient.prepareUpdate("photon", "place", String.valueOf(doc.getPlaceId())).setDoc(Utils.convert(doc, this.languages))); } catch (IOException e) { log.error(String.format("update of new doc [%s] failed", doc), e); } }
.prepareUpdate(indexName, typeName, id) .setDoc(jsonMap) .setUpsert(jsonMap)
private static void test() { for (int i = 0; i < 10; i++) { Map<String, Object> json = new HashMap<String, Object>(); json.put("field", "test"); addUpdateBuilderToBulk(client.prepareUpdate(Config.indexName, Config.typeName, String.valueOf(i)).setUpsert(json)); } System.out.println(bulkRequestBuilder.numberOfActions()); }
private UpdateRequestBuilder prepareIndexRequest(String indexName, DTO dto) throws JsonProcessingException { Map<String, Object> json = prepareDocument(dto); return this.client .prepareUpdate(indexName, this.configuration.getTypeFunction().apply(dto), String.valueOf(dto.getId())) .setDocAsUpsert(true).setDoc(this.objectWriter.writeValueAsString(json)); }
@Override public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit, Durability durability) throws IOException { try { String indexId = new String(put.getRow()); NavigableMap<byte[], List<Cell>> familyMap = put.getFamilyCellMap(); Map<String, Object> json = new HashMap<String, Object>(); for (Map.Entry<byte[], List<Cell>> entry : familyMap.entrySet()) { for (Cell cell : entry.getValue()) { String key = Bytes.toString(CellUtil.cloneQualifier(cell)); String value = Bytes.toString(CellUtil.cloneValue(cell)); json.put(key, value); } } ElasticSearchOperator.addUpdateBuilderToBulk(client.prepareUpdate(Config.indexName, Config.typeName, indexId).setUpsert(json)); LOG.info("observer -- add new doc: " + indexId + " to type: " + Config.typeName); } catch (Exception ex) { LOG.error(ex); } }
public Optional<BulkResponse> update(List<UpdatedRepresentation> updatedDocumentParts) { try { Preconditions.checkNotNull(updatedDocumentParts); BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); updatedDocumentParts.forEach(updatedDocumentPart -> bulkRequestBuilder.add( client.prepareUpdate( aliasName.getValue(), typeName.getValue(), updatedDocumentPart.getId()) .setDoc(updatedDocumentPart.getUpdatedDocumentPart()))); return Optional.of(bulkRequestBuilder.get()); } catch (ValidationException e) { LOGGER.warn("Error while updating index", e); return Optional.empty(); } }
private UpdateRequestBuilder prepareRemoveFieldsFromDocument(String indexName, String documentId, Collection<String> fields) { List<String> fieldNames = fields.stream().map(this::replaceFieldnameDots).collect(Collectors.toList()); if (fieldNames.isEmpty()) { return null; } return getClient().prepareUpdate() .setIndex(indexName) .setId(documentId) .setType(getIdStrategy().getType()) .setScript(new Script( ScriptType.STORED, "painless", "deleteFieldsFromDocumentScript", ImmutableMap.of("fieldNames", fieldNames) )) .setRetryOnConflict(MAX_RETRIES); }
private UpdateRequestBuilder prepareRemoveFieldsFromDocument(String indexName, String documentId, Collection<String> fields) { List<String> fieldNames = fields.stream().map(this::replaceFieldnameDots).collect(Collectors.toList()); if (fieldNames.isEmpty()) { return null; } return getClient().prepareUpdate() .setIndex(indexName) .setId(documentId) .setType(getIdStrategy().getType()) .setScript(new Script( ScriptType.STORED, "painless", "deleteFieldsFromDocumentScript", ImmutableMap.of("fieldNames", fieldNames) )) .setRetryOnConflict(MAX_RETRIES); }
/** * {@inheritDoc} */ @Override public void softDelete(final String type, final String id, final MetacatRequestContext metacatRequestContext) { try { RETRY_ES_PUBLISH.call(() -> { final XContentBuilder builder = XContentFactory.contentBuilder(contentType); builder.startObject().field(ElasticSearchDoc.Field.DELETED, true) .field(ElasticSearchDoc.Field.TIMESTAMP, java.time.Instant.now().toEpochMilli()) .field(ElasticSearchDoc.Field.USER, metacatRequestContext.getUserName()).endObject(); client.prepareUpdate(esIndex, type, id) .setRetryOnConflict(NO_OF_CONFLICT_RETRIES).setDoc(builder).get(esCallTimeout); ensureMigrationByCopy(type, Collections.singletonList(id)); return null; }); } catch (Exception e) { handleException("ElasticSearchUtil.softDelete", type, id, e, Metrics.CounterElasticSearchDelete.getMetricName()); } }
public void updateStatus(String url, String status) throws IOException { UpdateRequestBuilder update = getConnection().getClient() .prepareUpdate(getIndex(), getType(), url) .setDoc(jsonBuilder() .startObject() .field(STATUS_FIELD, status) .endObject()); getConnection().getProcessor().add(update.request()); }
public UpdateRequestBuilder updateIndexRequest( Event event,ElasticSearchEventSerializer elasticSearchEventSerializer) throws IOException { UpdateRequestBuilder indexRequestBuilder = null; XContentBuilder bytesStream = null; try { bytesStream = elasticSearchEventSerializer == null ?serializer.getContentBuilder(event):elasticSearchEventSerializer.getContentBuilder(event); indexRequestBuilder = client.prepareUpdate(event.getIndexPrefix(), event.getIndexType(),event.getId()) .setDoc(bytesStream); } finally { if (bytesStream != null) { // bytesStream.cl } } return indexRequestBuilder; }
@Override public void update(SearchDocument doc) throws IOException { ElasticsearchDocument esDoc = (ElasticsearchDocument)doc; bulkRequest.add(client.prepareUpdate(esDoc.getIndex(), esDoc.getType(), esDoc.getId()).setVersion( esDoc.getVersion()).setDoc(esDoc.getSource())); }
@Override public void alterElementVisibility( Graph graph, Element element, Visibility oldVisibility, Visibility newVisibility, Authorizations authorizations ) { String oldFieldName = deflatePropertyName(graph, ELEMENT_TYPE_FIELD_NAME, oldVisibility); Script script = new Script( "ctx._source.remove(oldFieldName);", ScriptService.ScriptType.INLINE, null, ImmutableMap.<String, Object>of("oldFieldName", oldFieldName) ); getClient().prepareUpdate() .setIndex(getIndexName(element)) .setId(element.getId()) .setType(ELEMENT_TYPE) .setScript(script) .get(); addElement(graph, element, authorizations); }
@Override public void markElementHidden(Graph graph, Element element, Visibility visibility, Authorizations authorizations) { try { String hiddenVisibilityPropertyName = addVisibilityToPropertyName(graph, HIDDEN_VERTEX_FIELD_NAME, visibility); if (!isPropertyInIndex(graph, hiddenVisibilityPropertyName)) { String indexName = getIndexName(element); IndexInfo indexInfo = ensureIndexCreatedAndInitialized(graph, indexName); addPropertyToIndex(graph, indexInfo, hiddenVisibilityPropertyName, visibility, Boolean.class, false); } XContentBuilder jsonBuilder = XContentFactory.jsonBuilder().startObject(); jsonBuilder.field(hiddenVisibilityPropertyName, true); jsonBuilder.endObject(); getClient() .prepareUpdate(getIndexName(element), ELEMENT_TYPE, element.getId()) .setDoc(jsonBuilder) .setRetryOnConflict(MAX_RETRIES) .get(); } catch (IOException e) { throw new VertexiumException("Could not mark element hidden", e); } }
@Override public void markPropertyHidden(Graph graph, Element element, Property property, Visibility visibility, Authorizations authorizations) { try { String hiddenVisibilityPropertyName = addVisibilityToPropertyName(graph, HIDDEN_PROPERTY_FIELD_NAME, visibility); String indexName = getIndexName(element); if (!isPropertyInIndex(graph, HIDDEN_PROPERTY_FIELD_NAME, visibility)) { IndexInfo indexInfo = ensureIndexCreatedAndInitialized(indexName); addPropertyToIndex(graph, indexInfo, hiddenVisibilityPropertyName, visibility, Boolean.class, false, false, false); } XContentBuilder jsonBuilder = XContentFactory.jsonBuilder().startObject(); jsonBuilder.field(hiddenVisibilityPropertyName, true); jsonBuilder.endObject(); getClient() .prepareUpdate(indexName, getIdStrategy().getType(), getIdStrategy().createElementDocId(element)) .setDoc(jsonBuilder) .setRetryOnConflict(MAX_RETRIES) .get(); getIndexRefreshTracker().pushChange(indexName); } catch (IOException e) { throw new VertexiumException("Could not mark element hidden", e); } }
private void updateDocs(final String type, final List<String> ids, final ObjectNode node) { try { RETRY_ES_PUBLISH.call(() -> { final BulkRequestBuilder bulkRequest = client.prepareBulk(); ids.forEach(id -> { bulkRequest.add(client.prepareUpdate(esIndex, type, id) .setRetryOnConflict(NO_OF_CONFLICT_RETRIES) .setDoc(metacatJson.toJsonAsBytes(node), XContentType.JSON)); }); final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout); if (bulkResponse.hasFailures()) { for (BulkItemResponse item : bulkResponse.getItems()) { if (item.isFailed()) { handleException("ElasticSearchUtil.updateDocs.item", type, item.getId(), item.getFailure().getCause(), Metrics.CounterElasticSearchUpdate.getMetricName()); } } } return null; }); } catch (Exception e) { handleException("ElasticSearchUtil.updatDocs", type, ids, e, Metrics.CounterElasticSearchBulkUpdate.getMetricName()); } }
private UpdateRequestBuilder prepareUpdate(Graph graph, Element element, Authorizations authorizations) { try { IndexInfo indexInfo = addPropertiesToIndex(graph, element, element.getProperties()); XContentBuilder source = buildJsonContentFromElement(graph, element, authorizations); if (MUTATION_LOGGER.isTraceEnabled()) { MUTATION_LOGGER.trace("addElement json: %s: %s", element.getId(), source.string()); } getIndexRefreshTracker().pushChange(indexInfo.getIndexName()); return getClient() .prepareUpdate(indexInfo.getIndexName(), getIdStrategy().getType(), getIdStrategy().createElementDocId(element)) .setDocAsUpsert(true) .setDoc(source) .setRetryOnConflict(MAX_RETRIES); } catch (IOException e) { throw new VertexiumException("Could not add element", e); } }
private UpdateRequestBuilder prepareUpdate(Graph graph, Element element, Authorizations authorizations) { try { IndexInfo indexInfo = addPropertiesToIndex(graph, element, element.getProperties()); XContentBuilder source = buildJsonContentFromElement(graph, element, authorizations); if (MUTATION_LOGGER.isTraceEnabled()) { MUTATION_LOGGER.trace("addElement json: %s: %s", element.getId(), source.string()); } getIndexRefreshTracker().pushChange(indexInfo.getIndexName()); return getClient() .prepareUpdate(indexInfo.getIndexName(), getIdStrategy().getType(), getIdStrategy().createElementDocId(element)) .setDocAsUpsert(true) .setDoc(source) .setRetryOnConflict(MAX_RETRIES); } catch (IOException e) { throw new VertexiumException("Could not add element", e); } }