@Override public Set<String> parseContext(Document document) { Set<String> values = null; if (fieldName != null) { IndexableField[] fields = document.getFields(fieldName); values = new HashSet<>(fields.length); for (IndexableField field : fields) { if (field instanceof SortedDocValuesField || field instanceof SortedSetDocValuesField || field instanceof StoredField) { // Ignore doc values and stored fields } else if (field.fieldType() instanceof KeywordFieldMapper.KeywordFieldType) { values.add(field.binaryValue().utf8ToString()); } else if (field.fieldType() instanceof StringFieldType) { values.add(field.stringValue()); } else { throw new IllegalArgumentException("Failed to parse context field [" + fieldName + "], only keyword and text fields are accepted"); } } } return (values == null) ? Collections.emptySet() : values; }
private static void addFields(ParseContext.Document nestedDoc, ParseContext.Document rootDoc) { for (IndexableField field : nestedDoc.getFields()) { if (!field.name().equals(UidFieldMapper.NAME) && !field.name().equals(TypeFieldMapper.NAME)) { rootDoc.add(field); } } }
IndexableField[] fields = document.getFields(fieldName); GeoPoint spare = new GeoPoint(); if (fields.length == 0) { IndexableField[] lonFields = document.getFields(fieldName + ".lon"); IndexableField[] latFields = document.getFields(fieldName + ".lat"); if (lonFields.length > 0 && latFields.length > 0) { for (int i = 0; i < lonFields.length; i++) {
private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException { // parse the document, at the moment we do update the mapping, just like percolate ParsedDocument parsedDocument = parseDocument(indexShard, indexShard.shardId().getIndexName(), request.type(), request.doc(), request.xContentType()); // select the right fields and generate term vectors ParseContext.Document doc = parsedDocument.rootDoc(); Set<String> seenFields = new HashSet<>(); Collection<DocumentField> documentFields = new HashSet<>(); for (IndexableField field : doc.getFields()) { MappedFieldType fieldType = indexShard.mapperService().fullName(field.name()); if (!isValidField(fieldType)) { continue; } if (request.selectedFields() != null && !request.selectedFields().contains(field.name())) { continue; } if (seenFields.contains(field.name())) { continue; } else { seenFields.add(field.name()); } String[] values = doc.getValues(field.name()); documentFields.add(new DocumentField(field.name(), Arrays.asList((Object[]) values))); } return generateTermVectors(indexShard, XContentHelper.convertToMap(parsedDocument.source(), true, request.xContentType()).v2(), documentFields, request.offsets(), request.perFieldAnalyzer(), seenFields); }
private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException { // parse the document, at the moment we do update the mapping, just like percolate ParsedDocument parsedDocument = parseDocument(indexShard, indexShard.shardId().getIndexName(), request.type(), request.doc(), request.xContentType()); // select the right fields and generate term vectors ParseContext.Document doc = parsedDocument.rootDoc(); Set<String> seenFields = new HashSet<>(); Collection<DocumentField> documentFields = new HashSet<>(); for (IndexableField field : doc.getFields()) { MappedFieldType fieldType = indexShard.mapperService().fullName(field.name()); if (!isValidField(fieldType)) { continue; } if (request.selectedFields() != null && !request.selectedFields().contains(field.name())) { continue; } if (seenFields.contains(field.name())) { continue; } else { seenFields.add(field.name()); } String[] values = doc.getValues(field.name()); documentFields.add(new DocumentField(field.name(), Arrays.asList((Object[]) values))); } return generateTermVectors(indexShard, XContentHelper.convertToMap(parsedDocument.source(), true, request.xContentType()).v2(), documentFields, request.offsets(), request.perFieldAnalyzer(), seenFields); }
@Override public void prepare(PercolateContext context, ParsedDocument parsedDocument) { MemoryIndex memoryIndex = cache.get(); for (IndexableField field : parsedDocument.rootDoc().getFields()) { if (field.fieldType().indexOptions() == IndexOptions.NONE && field.name().equals(UidFieldMapper.NAME)) { continue; } try { Analyzer analyzer = context.mapperService().documentMapper(parsedDocument.type()).mappers().indexAnalyzer(); // TODO: instead of passing null here, we can have a CTL<Map<String,TokenStream>> and pass previous, // like the indexer does try (TokenStream tokenStream = field.tokenStream(analyzer, null)) { if (tokenStream != null) { memoryIndex.addField(field.name(), tokenStream, field.boost()); } } } catch (Exception e) { throw new ElasticsearchException("Failed to create token stream for [" + field.name() + "]", e); } } context.initialize(new DocEngineSearcher(memoryIndex), parsedDocument); }
@Override public Set<CharSequence> parseContext(Document document) { Set<CharSequence> values = null; if (fieldName != null) { IndexableField[] fields = document.getFields(fieldName); values = new HashSet<>(fields.length); for (IndexableField field : fields) { if (field instanceof SortedDocValuesField || field instanceof SortedSetDocValuesField || field instanceof StoredField) { // Ignore doc values and stored fields } else if (field.fieldType() instanceof KeywordFieldMapper.KeywordFieldType) { values.add(field.binaryValue().utf8ToString()); } else if (field.fieldType() instanceof StringFieldType) { values.add(field.stringValue()); } else { throw new IllegalArgumentException("Failed to parse context field [" + fieldName + "], only keyword and text fields are accepted"); } } } return (values == null) ? Collections.emptySet() : values; }
MemoryIndex indexDoc(ParseContext.Document d, Analyzer analyzer, MemoryIndex memoryIndex) { for (IndexableField field : d.getFields()) { if (field.fieldType().indexOptions() == IndexOptions.NONE && field.name().equals(UidFieldMapper.NAME)) { continue; } try { // TODO: instead of passing null here, we can have a CTL<Map<String,TokenStream>> and pass previous, // like the indexer does try (TokenStream tokenStream = field.tokenStream(analyzer, null)) { if (tokenStream != null) { memoryIndex.addField(field.name(), tokenStream, field.boost()); } } } catch (IOException e) { throw new ElasticsearchException("Failed to create token stream", e); } } return memoryIndex; }
@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { if (fieldType().isEnabled() == false) { return; } for (ParseContext.Document document : context.docs()) { final List<String> paths = new ArrayList<>(); for (IndexableField field : document.getFields()) { paths.add(field.name()); } for (String path : paths) { for (String fieldName : extractFieldNames(path)) { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { document.add(new Field(fieldType().names().indexName(), fieldName, fieldType())); } } } } }
@Override public Set<CharSequence> parseContext(Document document) { Set<CharSequence> values = null; if (fieldName != null) { IndexableField[] fields = document.getFields(fieldName); values = new HashSet<>(fields.length); for (IndexableField field : fields) { if (field instanceof SortedDocValuesField || field instanceof SortedSetDocValuesField || field instanceof StoredField) { // Ignore doc values and stored fields } else if (field.fieldType() instanceof KeywordFieldMapper.KeywordFieldType) { values.add(field.binaryValue().utf8ToString()); } else if (field.fieldType() instanceof StringFieldType) { values.add(field.stringValue()); } else { throw new IllegalArgumentException("Failed to parse context field [" + fieldName + "], only keyword and text fields are accepted"); } } } return (values == null) ? Collections.emptySet() : values; }
@Override protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException { if (fieldType().isEnabled() == false) { return; } for (ParseContext.Document document : context.docs()) { final List<String> paths = new ArrayList<>(); for (IndexableField field : document.getFields()) { paths.add(field.name()); } for (String path : paths) { for (String fieldName : extractFieldNames(path)) { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { document.add(new Field(fieldType().name(), fieldName, fieldType())); } } } } }
@Override protected TokenStream wrapTokenStream(Document doc, TokenStream stream) { if (values != null) { return new PrefixAnalyzer.PrefixTokenFilter(stream, ContextMapping.SEPARATOR, values); // if fieldname is default, BUT our default values are set, we take that one } else if ((doc.getFields(fieldname).length == 0 || fieldname.equals(DEFAULT_FIELDNAME)) && defaultValues.iterator().hasNext()) { return new PrefixAnalyzer.PrefixTokenFilter(stream, ContextMapping.SEPARATOR, defaultValues); } else { IndexableField[] fields = doc.getFields(fieldname); ArrayList<CharSequence> values = new ArrayList<>(fields.length); for (int i = 0; i < fields.length; i++) { values.add(fields[i].stringValue()); } return new PrefixAnalyzer.PrefixTokenFilter(stream, ContextMapping.SEPARATOR, values); } }
@Override public Set<CharSequence> parseContext(Document document) { Set<CharSequence> values = null; if (fieldName != null) { IndexableField[] fields = document.getFields(fieldName); values = new HashSet<>(fields.length); for (IndexableField field : fields) { if (field.fieldType() instanceof KeywordFieldMapper.KeywordFieldType) { values.add(field.binaryValue().utf8ToString()); } else { values.add(field.stringValue()); } } } return (values == null) ? Collections.emptySet() : values; }
public void initialize(Engine.Searcher docSearcher, ParsedDocument parsedDocument) { this.docSearcher = docSearcher; IndexReader indexReader = docSearcher.reader(); LeafReaderContext atomicReaderContext = indexReader.leaves().get(0); LeafSearchLookup leafLookup = lookup().getLeafSearchLookup(atomicReaderContext); leafLookup.setDocument(0); leafLookup.source().setSource(parsedDocument.source()); Map<String, SearchHitField> fields = new HashMap<>(); for (IndexableField field : parsedDocument.rootDoc().getFields()) { fields.put(field.name(), new InternalSearchHitField(field.name(), Collections.emptyList())); } hitContext().reset( new InternalSearchHit(0, "unknown", new Text(parsedDocument.type()), fields), atomicReaderContext, 0, docSearcher.searcher() ); }
@Override protected TokenStream wrapTokenStream(Document doc, TokenStream stream) { if (values != null) { return new PrefixAnalyzer.PrefixTokenFilter(stream, ContextMapping.SEPARATOR, values); // if fieldname is default, BUT our default values are set, we take that one } else if ((doc.getFields(fieldname).length == 0 || fieldname.equals(DEFAULT_FIELDNAME)) && defaultValues.iterator().hasNext()) { return new PrefixAnalyzer.PrefixTokenFilter(stream, ContextMapping.SEPARATOR, defaultValues); } else { IndexableField[] fields = doc.getFields(fieldname); ArrayList<CharSequence> values = new ArrayList<>(fields.length); for (int i = 0; i < fields.length; i++) { values.add(fields[i].stringValue()); } return new PrefixAnalyzer.PrefixTokenFilter(stream, ContextMapping.SEPARATOR, values); } }
private static void addFields(ParseContext.Document nestedDoc, ParseContext.Document rootDoc) { for (IndexableField field : nestedDoc.getFields()) { if (!field.name().equals(UidFieldMapper.NAME) && !field.name().equals(TypeFieldMapper.NAME)) { rootDoc.add(field); } } }
private static void addFields(ParseContext.Document nestedDoc, ParseContext.Document rootDoc) { for (IndexableField field : nestedDoc.getFields()) { if (!field.name().equals(UidFieldMapper.NAME) && !field.name().equals(TypeFieldMapper.NAME)) { rootDoc.add(field); } } }
private static void addFields(ParseContext.Document nestedDoc, ParseContext.Document rootDoc) { for (IndexableField field : nestedDoc.getFields()) { if (!field.name().equals(UidFieldMapper.NAME) && !field.name().equals(TypeFieldMapper.NAME)) { rootDoc.add(field); } } }
private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException { // parse the document, at the moment we do update the mapping, just like percolate ParsedDocument parsedDocument = parseDocument(indexShard, indexShard.shardId().getIndexName(), request.type(), request.doc(), request.xContentType()); // select the right fields and generate term vectors ParseContext.Document doc = parsedDocument.rootDoc(); Set<String> seenFields = new HashSet<>(); Collection<DocumentField> documentFields = new HashSet<>(); for (IndexableField field : doc.getFields()) { MappedFieldType fieldType = indexShard.mapperService().fullName(field.name()); if (!isValidField(fieldType)) { continue; } if (request.selectedFields() != null && !request.selectedFields().contains(field.name())) { continue; } if (seenFields.contains(field.name())) { continue; } else { seenFields.add(field.name()); } String[] values = doc.getValues(field.name()); documentFields.add(new DocumentField(field.name(), Arrays.asList((Object[]) values))); } return generateTermVectors(indexShard, XContentHelper.convertToMap(parsedDocument.source(), true, request.xContentType()).v2(), documentFields, request.offsets(), request.perFieldAnalyzer(), seenFields); }
@Override protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException { if (fieldType().isEnabled() == false) { return; } for (ParseContext.Document document : context) { final List<String> paths = new ArrayList<>(document.getFields().size()); String previousPath = ""; // used as a sentinel - field names can't be empty for (IndexableField field : document.getFields()) { final String path = field.name(); if (path.equals(previousPath)) { // Sometimes mappers create multiple Lucene fields, eg. one for indexing, // one for doc values and one for storing. Deduplicating is not required // for correctness but this simple check helps save utf-8 conversions and // gives Lucene fewer values to deal with. continue; } paths.add(path); previousPath = path; } for (String path : paths) { for (String fieldName : extractFieldNames(path)) { if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { document.add(new Field(fieldType().name(), fieldName, fieldType())); } } } } }