private static ParsedDocument parsedDocument(SourceToParse source, ParseContext.InternalParseContext context, Mapping update) { return new ParsedDocument( context.version(), context.seqID(), context.sourceToParse().id(), context.sourceToParse().type(), source.routing(), context.docs(), context.sourceToParse().source(), context.sourceToParse().getXContentType(), update ).parent(source.parent()); }
public static SourceToParse source(String index, String type, String id, BytesReference source, XContentType contentType) { return new SourceToParse(index, type, id, source, contentType); }
@Override protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException { if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { Field uid = new Field(NAME, Uid.createUid(context.sourceToParse().type(), context.sourceToParse().id()), fieldType); fields.add(uid); } }
ParsedDocument parseDocument(SourceToParse source, MetadataFieldMapper[] metadataFieldsMappers) throws MapperParsingException { validateType(source); final Mapping mapping = docMapper.mapping(); final ParseContext.InternalParseContext context; final XContentType xContentType = source.getXContentType(); try (XContentParser parser = XContentHelper.createParser(docMapperParser.getXContentRegistry(), LoggingDeprecationHandler.INSTANCE, source.source(), xContentType)) { context = new ParseContext.InternalParseContext(indexSettings, docMapperParser, docMapper, source, parser); validateStart(parser); internalParseDocument(mapping, metadataFieldsMappers, context, parser); validateEnd(parser); } catch (Exception e) { throw wrapInMapperParsingException(source, e); } String remainingPath = context.path().pathAsText(""); if (remainingPath.isEmpty() == false) { throw new IllegalStateException("found leftover path elements: " + remainingPath); } context.postParse(); return parsedDocument(source, context, createDynamicUpdate(mapping, docMapper, context.getDynamicMappers())); }
/** * Utility method to create either an index or a create operation depending * on the {@link IndexRequest.OpType} of the request. */ public static Engine.IndexingOperation prepareIndexOperationOnPrimary(BulkShardRequest shardRequest, IndexRequest request, IndexShard indexShard) { SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, request.source()).index(request.index()).type(request.type()).id(request.id()) .routing(request.routing()).parent(request.parent()).timestamp(request.timestamp()).ttl(request.ttl()); boolean canHaveDuplicates = request.canHaveDuplicates(); if (shardRequest != null) { canHaveDuplicates |= shardRequest.canHaveDuplicates(); } if (request.opType() == IndexRequest.OpType.INDEX) { return indexShard.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType(), canHaveDuplicates); } else { assert request.opType() == IndexRequest.OpType.CREATE : request.opType(); return indexShard.prepareCreateOnPrimary(sourceToParse, request.version(), request.versionType(), canHaveDuplicates, request.autoGeneratedId()); } }
/** Executes index operation on primary shard after updates mapping if dynamic mappings are found */ private static void executeIndexRequestOnPrimary(BulkPrimaryExecutionContext context, MappingUpdatePerformer mappingUpdater) throws Exception { final IndexRequest request = context.getRequestToExecute(); final IndexShard primary = context.getPrimary(); final SourceToParse sourceToParse = SourceToParse.source(request.index(), request.type(), request.id(), request.source(), request.getContentType()) .routing(request.routing()).parent(request.parent()); executeOnPrimaryWhileHandlingMappingUpdates(context, () -> primary.applyIndexOperationOnPrimary(request.version(), request.versionType(), sourceToParse, request.ifSeqNo(), request.ifPrimaryTerm(), request.getAutoGeneratedTimestamp(), request.isRetry()), e -> primary.getFailedIndexResult(e, request.version()), context::markOperationAsExecuted, mapping -> mappingUpdater.updateMappings(mapping, primary.shardId(), request.type())); }
private static ParsedDocument parsedDocument(SourceToParse source, ParseContext.InternalParseContext context, Mapping update) { return new ParsedDocument( context.version(), context.sourceToParse().id(), context.sourceToParse().type(), source.routing(), source.timestamp(), source.ttl(), source.token(), context.docs(), context.sourceToParse().source(), context.sourceToParse().getXContentType(), update ).parent(source.parent()); }
/** Utility method to prepare an index operation on primary shards */ static Engine.Index prepareIndexOperationOnPrimary(IndexRequest request, IndexShard primary) { SourceToParse sourceToParse = SourceToParse.source(SourceToParse.Origin.PRIMARY, request.index(), request.type(), request.id(), request.source(), request.getContentType()).routing(request.routing()).parent(request.parent()) .timestamp(request.timestamp()).ttl(request.ttl()); return primary.prepareIndexOnPrimary(sourceToParse, request.version(), request.versionType(), request.getAutoGeneratedTimestamp(), request.isRetry()); }
private static MapperParsingException wrapInMapperParsingException(SourceToParse source, Exception e) { // if its already a mapper parsing exception, no need to wrap it... if (e instanceof MapperParsingException) { return (MapperParsingException) e; } // Throw a more meaningful message if the document is empty. if (source.source() != null && source.source().length() == 0) { return new MapperParsingException("failed to parse, document is empty"); } return new MapperParsingException("failed to parse", e); }
if (source.type() != null && !source.type().equals(docMapper.type())) { throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + docMapper.type() + "]"); source.type(docMapper.type()); XContentParser parser = source.parser(); try { if (parser == null) { parser = XContentHelper.createParser(source.source()); && source.parser() == null && parser != null) { if (source.source() != null && source.source().length() == 0) { throw new MapperParsingException("failed to parse, document is empty"); } finally { if (source.parser() == null && parser != null) { parser.close(); ParsedDocument doc = new ParsedDocument(context.uid(), context.version(), context.id(), context.type(), source.routing(), source.timestamp(), source.ttl(), context.docs(), context.source(), update).parent(source.parent());
@Override protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException { boolean parent = context.docMapper().isParent(context.sourceToParse().type()); if (parent) { fields.add(new SortedDocValuesField(parentJoinField.fieldType().name(), new BytesRef(context.sourceToParse().id()))); context.sourceToParse().parent(parentId); fields.add(new SortedDocValuesField(fieldType.name(), new BytesRef(parentId))); } else { if (context.sourceToParse().parent() != null) { String parentId = context.sourceToParse().parent(); if (parsedParentId == null) { if (parentId == null) {
@Override protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException, AlreadyExpiredException { if (enabledState.enabled) { long ttl = context.sourceToParse().ttl(); if (ttl <= 0 && defaultTTL > 0) { // no ttl provided so we use the default value ttl = defaultTTL; context.sourceToParse().ttl(ttl); } if (ttl > 0) { // a ttl has been provided either externally or in the _source long timestamp = context.sourceToParse().timestamp(); long expire = new Date(timestamp + ttl).getTime(); long now = System.currentTimeMillis(); // there is not point indexing already expired doc if (context.sourceToParse().origin() == SourceToParse.Origin.PRIMARY && now >= expire) { throw new AlreadyExpiredException(context.sourceToParse().index(), context.sourceToParse().type(), context.sourceToParse().id(), timestamp, ttl, now); } // the expiration timestamp (timestamp + ttl) is set as field fields.add(new LegacyLongFieldMapper.CustomLongNumericField(expire, fieldType())); } } }
throw new IllegalArgumentException("[parent] is missing for join field [" + name() + "]"); if (context.sourceToParse().routing() == null) { throw new IllegalArgumentException("[routing] is missing for join field [" + name() + "]"); ParseContext externalContext = context.createExternalValueContext(context.sourceToParse().id()); parentIdField.parse(externalContext);
private ParsedDocument parseDocument(String index, String type, BytesReference doc) throws Throwable { MapperService mapperService = indexShard.mapperService(); // TODO: make parsing not dynamically create fields not in the original mapping DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(type); ParsedDocument parsedDocument = docMapper.getDocumentMapper().parse(source(doc).index(index).type(type).flyweight(true)); if (docMapper.getMapping() != null) { parsedDocument.addDynamicMappingsUpdate(docMapper.getMapping()); } if (parsedDocument.dynamicMappingsUpdate() != null) { mappingUpdatedAction.updateMappingOnMasterSynchronously(index, type, parsedDocument.dynamicMappingsUpdate()); } return parsedDocument; }
@Override protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException { if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) { if (context.mapperService().getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_0_0_beta1)) { BytesRef id = Uid.encodeId(context.sourceToParse().id()); fields.add(new Field(NAME, id, fieldType)); } else { fields.add(new Field(NAME, context.sourceToParse().id(), fieldType)); } } }
@Override protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException { String routing = context.sourceToParse().routing(); if (routing != null) { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { fields.add(new Field(fieldType().name(), routing, fieldType())); createFieldNamesField(context, fields); } } }
private void validateType(SourceToParse source) { if (docMapper.type().equals(MapperService.DEFAULT_MAPPING)) { throw new IllegalArgumentException("It is forbidden to index into the default mapping [" + MapperService.DEFAULT_MAPPING + "]"); } if (Objects.equals(source.type(), docMapper.type()) == false) { throw new MapperParsingException("Type mismatch, provide type [" + source.type() + "] but mapper is of type [" + docMapper.type() + "]"); } }
public Engine.Result applyTranslogOperation(Translog.Operation operation, Engine.Operation.Origin origin) throws IOException { final Engine.Result result; switch (operation.opType()) { case INDEX: final Translog.Index index = (Translog.Index) operation; // we set canHaveDuplicates to true all the time such that we de-optimze the translog case and ensure that all // autoGeneratedID docs that are coming from the primary are updated correctly. result = applyIndexOperation(index.seqNo(), index.primaryTerm(), index.version(), index.versionType().versionTypeForReplicationAndRecovery(), index.getAutoGeneratedIdTimestamp(), true, origin, source(shardId.getIndexName(), index.type(), index.id(), index.source(), XContentHelper.xContentType(index.source())).routing(index.routing()).parent(index.parent())); break; case DELETE: final Translog.Delete delete = (Translog.Delete) operation; result = applyDeleteOperation(delete.seqNo(), delete.primaryTerm(), delete.version(), delete.type(), delete.id(), delete.versionType().versionTypeForReplicationAndRecovery(), origin); break; case NO_OP: final Translog.NoOp noOp = (Translog.NoOp) operation; result = markSeqNoAsNoop(noOp.seqNo(), noOp.primaryTerm(), noOp.reason(), origin); break; default: throw new IllegalStateException("No operation defined for [" + operation + "]"); } return result; }
@Override public void postParse(ParseContext context) throws IOException { if (context.id() == null && !context.sourceToParse().flyweight()) { throw new MapperParsingException("No id found while parsing the content source"); } // if we did not have the id as part of the sourceToParse, then we need to parse it here // it would have been filled in the _id parse phase if (context.sourceToParse().id() == null) { super.parse(context); // since we did not have the uid in the pre phase, we did not add it automatically to the nested docs // as they were created we need to make sure we add it to all the nested docs... if (context.docs().size() > 1) { final IndexableField uidField = context.rootDoc().getField(UidFieldMapper.NAME); assert uidField != null; // we need to go over the docs and add it... for (int i = 1; i < context.docs().size(); i++) { final Document doc = context.docs().get(i); doc.add(new Field(UidFieldMapper.NAME, uidField.stringValue(), Defaults.NESTED_FIELD_TYPE)); } } } }