if (Strings.hasLength(indexRequest.getPipeline())) { metadata.field("pipeline", indexRequest.getPipeline());
public static Request index(IndexRequest indexRequest) { String method = Strings.hasLength(indexRequest.id()) ? HttpMethod.PUT.name() : HttpMethod.POST.name(); boolean isCreate = (indexRequest.opType() == DocWriteRequest.OpType.CREATE); String endpoint = endpoint(indexRequest.index(), indexRequest.type(), indexRequest.id(), isCreate ? "_create" : null); Request request = new Request(method, endpoint); Params parameters = new Params(request); parameters.withRouting(indexRequest.routing()); parameters.withTimeout(indexRequest.timeout()); parameters.withVersion(indexRequest.version()); parameters.withVersionType(indexRequest.versionType()); parameters.withPipeline(indexRequest.getPipeline()); parameters.withRefreshPolicy(indexRequest.getRefreshPolicy()); parameters.withWaitForActiveShards(indexRequest.waitForActiveShards()); BytesRef source = indexRequest.source().toBytesRef(); ContentType contentType = createContentType(indexRequest.getContentType()); request.setEntity(new ByteArrayEntity(source.bytes, source.offset, source.length, contentType)); return request; }
@Override protected void doRun() { for (DocWriteRequest<?> actionRequest : actionRequests) { IndexRequest indexRequest = TransportBulkAction.getIndexWriteRequest(actionRequest); if (indexRequest == null) { continue; } String pipelineId = indexRequest.getPipeline(); if (NOOP_PIPELINE_NAME.equals(pipelineId) == false) { try { Pipeline pipeline = pipelines.get(pipelineId); if (pipeline == null) { throw new IllegalArgumentException("pipeline with id [" + pipelineId + "] does not exist"); } innerExecute(indexRequest, pipeline, itemDroppedHandler); //this shouldn't be needed here but we do it for consistency with index api // which requires it to prevent double execution indexRequest.setPipeline(NOOP_PIPELINE_NAME); } catch (Exception e) { itemFailureHandler.accept(indexRequest, e); } } } completionHandler.accept(null); } });
void processBulkIndexIngestRequest(Task task, BulkRequest original, ActionListener<BulkResponse> listener) { long ingestStartTimeInNanos = System.nanoTime(); BulkRequestModifier bulkRequestModifier = new BulkRequestModifier(original); ingestService.executeBulkRequest(() -> bulkRequestModifier, (indexRequest, exception) -> { logger.debug(() -> new ParameterizedMessage("failed to execute pipeline [{}] for document [{}/{}/{}]", indexRequest.getPipeline(), indexRequest.index(), indexRequest.type(), indexRequest.id()), exception); bulkRequestModifier.markCurrentItemAsFailed(exception); }, (exception) -> { if (exception != null) { logger.error("failed to execute pipeline for a bulk request", exception); listener.onFailure(exception); } else { long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ingestStartTimeInNanos); BulkRequest bulkRequest = bulkRequestModifier.getBulkRequest(); ActionListener<BulkResponse> actionListener = bulkRequestModifier.wrapActionListenerIfNeeded(ingestTookInMillis, listener); if (bulkRequest.requests().isEmpty()) { // at this stage, the transport bulk action can't deal with a bulk request with no requests, // so we stop and send an empty response back to the client. // (this will happen if pre-processing all items in the bulk failed) actionListener.onResponse(new BulkResponse(new BulkItemResponse[0], 0)); } else { doExecute(task, bulkRequest, actionListener); } } }, indexRequest -> bulkRequestModifier.markCurrentItemAsDropped()); }
IndexRequest indexRequest = getIndexWriteRequest(actionRequest); if(indexRequest != null){ String pipeline = indexRequest.getPipeline(); if (pipeline == null) { IndexMetaData indexMetaData = indicesMetaData.get(actionRequest.index());
if (getDestination().getPipeline() != null) { builder.field("pipeline", getDestination().getPipeline());
@Override protected void doRun() throws Exception { for (DocWriteRequest actionRequest : actionRequests) { if ((actionRequest instanceof IndexRequest)) { IndexRequest indexRequest = (IndexRequest) actionRequest; if (Strings.hasText(indexRequest.getPipeline())) { try { innerExecute(indexRequest, getPipeline(indexRequest.getPipeline())); //this shouldn't be needed here but we do it for consistency with index api // which requires it to prevent double execution indexRequest.setPipeline(null); } catch (Exception e) { itemFailureHandler.accept(indexRequest, e); } } } } completionHandler.accept(null); } });
/** * @return Whether this bulk request contains index request with an ingest pipeline enabled. */ public boolean hasIndexRequestsWithPipelines() { for (DocWriteRequest actionRequest : requests) { if (actionRequest instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) actionRequest; if (Strings.hasText(indexRequest.getPipeline())) { return true; } } } return false; }
public void executeIndexRequest(IndexRequest request, Consumer<Exception> failureHandler, Consumer<Boolean> completionHandler) { Pipeline pipeline = getPipeline(request.getPipeline()); threadPool.executor(ThreadPool.Names.INDEX).execute(new AbstractRunnable() { @Override public void onFailure(Exception e) { failureHandler.accept(e); } @Override protected void doRun() throws Exception { innerExecute(request, pipeline); completionHandler.accept(true); } }); }
@Override protected void doRun() throws Exception { for (DocWriteRequest actionRequest : actionRequests) { IndexRequest indexRequest = null; if (actionRequest instanceof IndexRequest) { indexRequest = (IndexRequest) actionRequest; } else if (actionRequest instanceof UpdateRequest) { UpdateRequest updateRequest = (UpdateRequest) actionRequest; indexRequest = updateRequest.docAsUpsert() ? updateRequest.doc() : updateRequest.upsertRequest(); } if (indexRequest != null && Strings.hasText(indexRequest.getPipeline())) { try { innerExecute(indexRequest, getPipeline(indexRequest.getPipeline())); //this shouldn't be needed here but we do it for consistency with index api // which requires it to prevent double execution indexRequest.setPipeline(null); } catch (Exception e) { itemFailureHandler.accept(indexRequest, e); } } } completionHandler.accept(null); } });
continue; String pipelineId = indexRequest.getPipeline(); if (NOOP_PIPELINE_NAME.equals(pipelineId) == false) { try {
void processBulkIndexIngestRequest(Task task, BulkRequest original, ActionListener<BulkResponse> listener) { long ingestStartTimeInNanos = System.nanoTime(); BulkRequestModifier bulkRequestModifier = new BulkRequestModifier(original); ingestService.getPipelineExecutionService().executeBulkRequest(() -> bulkRequestModifier, (indexRequest, exception) -> { logger.debug((Supplier<?>) () -> new ParameterizedMessage("failed to execute pipeline [{}] for document [{}/{}/{}]", indexRequest.getPipeline(), indexRequest.index(), indexRequest.type(), indexRequest.id()), exception); bulkRequestModifier.markCurrentItemAsFailed(exception); }, (exception) -> { if (exception != null) { logger.error("failed to execute pipeline for a bulk request", exception); listener.onFailure(exception); } else { long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ingestStartTimeInNanos); BulkRequest bulkRequest = bulkRequestModifier.getBulkRequest(); ActionListener<BulkResponse> actionListener = bulkRequestModifier.wrapActionListenerIfNeeded(ingestTookInMillis, listener); if (bulkRequest.requests().isEmpty()) { // at this stage, the transport bulk action can't deal with a bulk request with no requests, // so we stop and send an empty response back to the client. // (this will happen if pre-processing all items in the bulk failed) actionListener.onResponse(new BulkResponse(new BulkItemResponse[0], 0)); } else { doExecute(task, bulkRequest, actionListener); } } }); }
void processBulkIndexIngestRequest(Task task, BulkRequest original, ActionListener<BulkResponse> listener) { long ingestStartTimeInNanos = System.nanoTime(); BulkRequestModifier bulkRequestModifier = new BulkRequestModifier(original); ingestService.executeBulkRequest(() -> bulkRequestModifier, (indexRequest, exception) -> { logger.debug(() -> new ParameterizedMessage("failed to execute pipeline [{}] for document [{}/{}/{}]", indexRequest.getPipeline(), indexRequest.index(), indexRequest.type(), indexRequest.id()), exception); bulkRequestModifier.markCurrentItemAsFailed(exception); }, (exception) -> { if (exception != null) { logger.error("failed to execute pipeline for a bulk request", exception); listener.onFailure(exception); } else { long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ingestStartTimeInNanos); BulkRequest bulkRequest = bulkRequestModifier.getBulkRequest(); ActionListener<BulkResponse> actionListener = bulkRequestModifier.wrapActionListenerIfNeeded(ingestTookInMillis, listener); if (bulkRequest.requests().isEmpty()) { // at this stage, the transport bulk action can't deal with a bulk request with no requests, // so we stop and send an empty response back to the client. // (this will happen if pre-processing all items in the bulk failed) actionListener.onResponse(new BulkResponse(new BulkItemResponse[0], 0)); } else { doExecute(task, bulkRequest, actionListener); } } }, indexRequest -> bulkRequestModifier.markCurrentItemAsDropped()); }
if (actionRequest instanceof IndexRequest) { IndexRequest indexRequest = (IndexRequest) actionRequest; String pipeline = indexRequest.getPipeline(); if (pipeline == null) { IndexMetaData indexMetaData = indicesMetaData.get(indexRequest.index());
if (getDestination().getPipeline() != null) { builder.field("pipeline", getDestination().getPipeline());
if (Strings.hasLength(indexRequest.getPipeline())) { metadata.field("pipeline", indexRequest.getPipeline());
if (Strings.hasLength(indexRequest.getPipeline())) { metadata.field("pipeline", indexRequest.getPipeline());
static Request index(IndexRequest indexRequest) { String method = Strings.hasLength(indexRequest.id()) ? HttpPut.METHOD_NAME : HttpPost.METHOD_NAME; boolean isCreate = (indexRequest.opType() == DocWriteRequest.OpType.CREATE); String endpoint = endpoint(indexRequest.index(), indexRequest.type(), indexRequest.id(), isCreate ? "_create" : null); Request request = new Request(method, endpoint); Params parameters = new Params(request); parameters.withRouting(indexRequest.routing()); parameters.withParent(indexRequest.parent()); parameters.withTimeout(indexRequest.timeout()); parameters.withVersion(indexRequest.version()); parameters.withVersionType(indexRequest.versionType()); parameters.withPipeline(indexRequest.getPipeline()); parameters.withRefreshPolicy(indexRequest.getRefreshPolicy()); parameters.withWaitForActiveShards(indexRequest.waitForActiveShards(), ActiveShardCount.DEFAULT); BytesRef source = indexRequest.source().toBytesRef(); ContentType contentType = createContentType(indexRequest.getContentType()); request.setEntity(new ByteArrayEntity(source.bytes, source.offset, source.length, contentType)); return request; }
static Request index(IndexRequest indexRequest) { String method = Strings.hasLength(indexRequest.id()) ? HttpPut.METHOD_NAME : HttpPost.METHOD_NAME; boolean isCreate = (indexRequest.opType() == DocWriteRequest.OpType.CREATE); String endpoint = endpoint(indexRequest.index(), indexRequest.type(), indexRequest.id(), isCreate ? "_create" : null); Params parameters = Params.builder(); parameters.withRouting(indexRequest.routing()); parameters.withParent(indexRequest.parent()); parameters.withTimeout(indexRequest.timeout()); parameters.withVersion(indexRequest.version()); parameters.withVersionType(indexRequest.versionType()); parameters.withPipeline(indexRequest.getPipeline()); parameters.withRefreshPolicy(indexRequest.getRefreshPolicy()); parameters.withWaitForActiveShards(indexRequest.waitForActiveShards(), ActiveShardCount.DEFAULT); BytesRef source = indexRequest.source().toBytesRef(); ContentType contentType = createContentType(indexRequest.getContentType()); HttpEntity entity = new ByteArrayEntity(source.bytes, source.offset, source.length, contentType); return new Request(method, endpoint, parameters.getParams(), entity); }
index.setPipeline(mainRequest.getDestination().getPipeline());