private IndexRequest createRequest(D document, String index) { if(document.getTimestamp() == null) { throw new IllegalArgumentException("Document must contain the timestamp"); } // if updating an existing document, the doc ID should be defined. // if creating a new document, set the doc ID to null to allow Elasticsearch to generate one. String docId = document.getDocumentID().orElse(null); if(LOG.isDebugEnabled() && document.getDocumentID().isPresent()) { LOG.debug("Updating existing document with known doc ID; docID={}, guid={}, sensorType={}", docId, document.getGuid(), document.getSensorType()); } else if(LOG.isDebugEnabled()) { LOG.debug("Creating a new document, doc ID not yet known; guid={}, sensorType={}", document.getGuid(), document.getSensorType()); } return new IndexRequest() .source(document.getDocument()) .type(document.getSensorType() + "_doc") .index(index) .id(docId) .index(index) .timestamp(document.getTimestamp().toString()); }
/** * Sets the timestamp either as millis since the epoch, or, in the configured date format. */ public IndexRequestBuilder setTimestamp(String timestamp) { request.timestamp(timestamp); return this; }
public IndexRequestBuilder<JsonInput, JsonOutput> timestamp(String timestamp) { request.timestamp(timestamp); return this; }
@Override protected void scriptChangedTimestamp(IndexRequest index, Object to) { index.timestamp(Objects.toString(to, null)); }
/** * Sets the timestamp either as millis since the epoch, or, in the configured date format. */ @Deprecated public IndexRequestBuilder setTimestamp(String timestamp) { request.timestamp(timestamp); return this; }
@Override protected void scriptChangedTimestamp(IndexRequest index, Object to) { index.timestamp(Objects.toString(to, null)); }
/** * Copies the metadata from a hit to the index request. */ protected void copyMetadata(IndexRequest index, SearchHit doc) { index.parent(this.<String>fieldValue(doc, ParentFieldMapper.NAME)); copyRouting(index, doc); // Comes back as a Long but needs to be a string Long timestamp = fieldValue(doc, TimestampFieldMapper.NAME); if (timestamp != null) { index.timestamp(timestamp.toString()); } Long ttl = fieldValue(doc, TTLFieldMapper.NAME); if (ttl != null) { index.ttl(ttl); } }
/** * Copies the metadata from a hit to the index request. */ protected void copyMetadata(IndexRequest index, SearchHit doc) { index.parent(this.<String>fieldValue(doc, ParentFieldMapper.NAME)); copyRouting(index, doc); // Comes back as a Long but needs to be a string Long timestamp = fieldValue(doc, TimestampFieldMapper.NAME); if (timestamp != null) { index.timestamp(timestamp.toString()); } Long ttl = fieldValue(doc, TTLFieldMapper.NAME); if (ttl != null) { index.ttl(ttl); } }
/** Utility method to prepare an index operation on primary shards */ static Engine.Index prepareIndexOperationOnPrimary(IndexRequest request, IndexShard primary) { SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, request.index(), request.type(), request.id(), request.source(), request.getContentType()).routing(request.routing()).parent(request.parent()) .timestamp(request.timestamp()).ttl(request.ttl()); return primary.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType(), request.getAutoGeneratedTimestamp(), request.isRetry()); }
e = addValidationError("setting ttl on destination isn't supported. use scripts instead.", e); if (destination.timestamp() != null) { e = addValidationError("setting timestamp on destination isn't supported. use scripts instead.", e);
e = addValidationError("setting ttl on destination isn't supported. use scripts instead.", e); if (destination.timestamp() != null) { e = addValidationError("setting timestamp on destination isn't supported. use scripts instead.", e);
/** * Execute the given {@link IndexRequest} on a replica shard, throwing a * {@link RetryOnReplicaException} if the operation needs to be re-tried. */ public static Engine.IndexResult executeIndexRequestOnReplica(IndexRequest request, IndexShard replica) throws IOException { final ShardId shardId = replica.shardId(); SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, shardId.getIndexName(), request.type(), request.id(), request.source(), request.getContentType()).routing(request.routing()).parent(request.parent()) .timestamp(request.timestamp()).ttl(request.ttl()); final Engine.Index operation; try { operation = replica.prepareIndexOnReplica(sourceToParse, request.version(), request.versionType(), request.getAutoGeneratedTimestamp(), request.isRetry()); } catch (MapperParsingException e) { return new Engine.IndexResult(e, request.version()); } Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); if (update != null) { throw new RetryOnReplicaException(shardId, "Mappings are not available on the replica yet, triggered update: " + update); } return replica.index(operation); }
break; case "timestamp": // This isn't actually supported but the validator will catch it and make a nice error message index.timestamp(parser.text()); break; case "ttl": // This isn't actually supported but the validator will catch it and make a nice error message
/** * Utility method to create either an index or a create operation depending * on the {@link IndexRequest.OpType} of the request. */ public static Engine.IndexingOperation prepareIndexOperationOnPrimary(BulkShardRequest shardRequest, IndexRequest request, IndexShard indexShard) { SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, request.source()).index(request.index()).type(request.type()).id(request.id()) .routing(request.routing()).parent(request.parent()).timestamp(request.timestamp()).ttl(request.ttl()); boolean canHaveDuplicates = request.canHaveDuplicates(); if (shardRequest != null) { canHaveDuplicates |= shardRequest.canHaveDuplicates(); } if (request.opType() == IndexRequest.OpType.INDEX) { return indexShard.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType(), canHaveDuplicates); } else { assert request.opType() == IndexRequest.OpType.CREATE : request.opType(); return indexShard.prepareCreateOnPrimary(sourceToParse, request.version(), request.versionType(), canHaveDuplicates, request.autoGeneratedId()); } }
/** * Execute the given {@link IndexRequest} on a replica shard, throwing a * {@link RetryOnReplicaException} if the operation needs to be re-tried. */ public static Engine.IndexingOperation executeIndexRequestOnReplica(IndexRequest request, IndexShard indexShard) { final ShardId shardId = indexShard.shardId(); SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.REPLICA, request.source()).index(shardId.getIndex()).type(request.type()).id(request.id()) .routing(request.routing()).parent(request.parent()).timestamp(request.timestamp()).ttl(request.ttl()); final Engine.IndexingOperation operation; if (request.opType() == IndexRequest.OpType.INDEX) { operation = indexShard.prepareIndexOnReplica(sourceToParse, request.version(), request.versionType(), request.canHaveDuplicates()); } else { assert request.opType() == IndexRequest.OpType.CREATE : request.opType(); operation = indexShard.prepareCreateOnReplica(sourceToParse, request.version(), request.versionType(), request.canHaveDuplicates(), request.autoGeneratedId()); } Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); if (update != null) { throw new RetryOnReplicaException(shardId, "Mappings are not available on the replica yet, triggered update: " + update); } operation.execute(indexShard); return operation; }
@Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { IndexRequest indexRequest = new IndexRequest(request.param("index"), request.param("type"), request.param("id")); indexRequest.routing(request.param("routing")); indexRequest.parent(request.param("parent")); // order is important, set it after routing, so it will set the routing if (request.hasParam("timestamp")) { deprecationLogger.deprecated("The [timestamp] parameter of index requests is deprecated"); } indexRequest.timestamp(request.param("timestamp")); if (request.hasParam("ttl")) { deprecationLogger.deprecated("The [ttl] parameter of index requests is deprecated"); indexRequest.ttl(request.param("ttl")); } indexRequest.setPipeline(request.param("pipeline")); indexRequest.source(request.requiredContent(), request.getXContentType()); indexRequest.timeout(request.paramAsTime("timeout", IndexRequest.DEFAULT_TIMEOUT)); indexRequest.setRefreshPolicy(request.param("refresh")); indexRequest.version(RestActions.parseVersion(request)); indexRequest.versionType(VersionType.fromString(request.param("version_type"), indexRequest.versionType())); String sOpType = request.param("op_type"); String waitForActiveShards = request.param("wait_for_active_shards"); if (waitForActiveShards != null) { indexRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); } if (sOpType != null) { indexRequest.opType(sOpType); } return channel -> client.index(indexRequest, new RestStatusToXContentListener<>(channel, r -> r.getLocation(indexRequest.routing()))); }
@Override public ListenableFuture<ResponseType> act(IndexRequest request) { UrlBuilder url = UrlBuilder.create() .protocol(protocol).host(host).port(port) .path(urlEncode(notNull(request.index()))) .seg(urlEncode(notNull(request.type()))) .paramIfPresent("routing", fromNullable(request.routing())) .paramIfPresent("parent", fromNullable(request.parent())) .paramIfPresent("timestamp", fromNullable(request.timestamp())) .paramIfPresent("ttl", (request.ttl() == -1) ? Optional.<String>absent() : of(Long.toString(request.ttl()))) .paramIfPresent("timeout", fromNullable(request.timeout()).transform(timeValueToString)) .paramIfPresent("refresh", fromNullable(request.refresh()).transform(booleanToString)) .paramIfPresent("version", fromNullable(request.version()).transform(longToString)) .paramIfPresent("version_type", fromNullable(request.versionType()).transform(versionTypeToString)) .paramIfPresent("op_type", fromNullable(request.opType()).transform(opTypeToString)) .paramIfPresent("replication", fromNullable(request.replicationType()).transform(replicationTypeToString)) .paramIfPresent("consistency", fromNullable(request.consistencyLevel()).transform(writeConsistencyLevelToString) ); // source: if (request.id() == null) { // auto id creation return Futures.transform(executor.post(url.url(), InputStreams.of(request.safeSource())), responseTransform); } else { return Futures.transform(executor.put(url.seg(urlEncode(request.id())).url(), InputStreams.of(request.safeSource())), responseTransform); } } }
@Override public ListenableFuture<ResponseType> act(IndexRequest request) { UrlBuilder url = UrlBuilder.create() .protocol(protocol).host(host).port(port) .path(urlEncode(notNull(request.index()))) .seg(urlEncode(notNull(request.type()))) .paramIfPresent("routing", fromNullable(request.routing())) .paramIfPresent("parent", fromNullable(request.parent())) .paramIfPresent("timestamp", fromNullable(request.timestamp())) .paramIfPresent("ttl", (request.ttl() == -1) ? Optional.<String>absent() : of(Long.toString(request.ttl()))) .paramIfPresent("timeout", fromNullable(request.timeout()).transform(timeValueToString)) .paramIfPresent("refresh", fromNullable(request.refresh()).transform(booleanToString)) .paramIfPresent("version", fromNullable(request.version()).transform(longToString)) .paramIfPresent("version_type", fromNullable(request.versionType()).transform(versionTypeToString)) .paramIfPresent("op_type", fromNullable(request.opType()).transform(opTypeToString)) .paramIfPresent("replication", fromNullable(request.replicationType()).transform(replicationTypeToString)) .paramIfPresent("consistency", fromNullable(request.consistencyLevel()).transform(writeConsistencyLevelToString) ); // source: if (request.id() == null) { // auto id creation return Futures.transform(executor.post(url.url(), InputStreams.of(request.safeSource())), responseTransform); } else { return Futures.transform(executor.put(url.seg(urlEncode(request.id())).url(), InputStreams.of(request.safeSource())), responseTransform); } } }