} else { fields.add(new BinaryDocValuesField(fieldName, new BytesRef((byte[]) document.get(fieldName))));
public boolean apply(IndexableField input) { // when applying filter for static columns, update _id and _uid.... if (input.name().equals(IdFieldMapper.NAME)) { ((Field)input).setStringValue(uid.id()); } if (input.name().equals(UidFieldMapper.NAME)) { if (input instanceof BinaryDocValuesField) { ((BinaryDocValuesField)input).setBytesValue(new BytesRef(uid.toString())); } else if (input instanceof Field) { ((Field)input).setStringValue(uid.toString()); } } if (input.name().startsWith("_")) { return true; } int x = input.name().indexOf('.'); String colName = (x > 0) ? input.name().substring(0,x) : input.name(); int idx = indexInfo.indexOf(colName); return idx < baseCfs.metadata.partitionKeyColumns().size() || indexInfo.isStaticField(idx) ; } }
} else { fields.add(new BinaryDocValuesField(fieldName, new BytesRef((byte[]) document.get(fieldName))));
} else { vector = new BinaryDocValuesField(extractorItems.get(extractorItem)[0], new BytesRef(globalFeature.getByteArrayRepresentation()));
} else { vector = new BinaryDocValuesField(extractorItems.get(extractorItem)[0], new BytesRef(globalFeature.getByteArrayRepresentation()));
private Document buildDocument(BytesRef text, Set<BytesRef> contexts, long weight, BytesRef payload) throws IOException { String textString = text.utf8ToString(); Document doc = new Document(); FieldType ft = getTextFieldType(); doc.add(new Field(TEXT_FIELD_NAME, textString, ft)); doc.add(new Field("textgrams", textString, ft)); doc.add(new StringField(EXACT_TEXT_FIELD_NAME, textString, Field.Store.NO)); doc.add(new BinaryDocValuesField(TEXT_FIELD_NAME, text)); doc.add(new NumericDocValuesField("weight", weight)); if (payload != null) { doc.add(new BinaryDocValuesField("payloads", payload)); } if (contexts != null) { for(BytesRef context : contexts) { doc.add(new StringField(CONTEXTS_FIELD_NAME, context, Field.Store.NO)); doc.add(new SortedSetDocValuesField(CONTEXTS_FIELD_NAME, context)); } } return doc; }
@Override public Void visit(Relation relation) { Document document = new Document(); document.add(new Field("class", "relation", classField)); document.add(new BinaryDocValuesField("class_value", class_relationByteRef)); document.add(new Field("relation.identity", String.valueOf(relation.getId()), classField)); document.add(new NumericDocValuesField("relation.identity_value", relation.getId())); // todo envelope addObjectFields(relation, document); try { indexWriter.updateDocument(new Term("relation.identity", String.valueOf(relation.getId())), document); } catch (IOException e) { throw new RuntimeException(e); } return null; } }
@Override final protected void newField(final String fieldName, final Object value, final FieldConsumer consumer) { consumer.accept(genericFieldName, fieldName, new BinaryDocValuesField(fieldName, new BytesRef(value.toString()))); }
@Override public Field[] createIndexableFields(Shape shape) { int bufSize = Math.max(128, (int) (this.indexLastBufSize * 1.5));//50% headroom over last ByteArrayOutputStream byteStream = new ByteArrayOutputStream(bufSize); final BytesRef bytesRef = new BytesRef();//receiver of byteStream's bytes try { ctx.getBinaryCodec().writeShape(new DataOutputStream(byteStream), shape); //this is a hack to avoid redundant byte array copying by byteStream.toByteArray() byteStream.writeTo(new FilterOutputStream(null/*not used*/) { @Override public void write(byte[] b, int off, int len) throws IOException { bytesRef.bytes = b; bytesRef.offset = off; bytesRef.length = len; } }); } catch (IOException e) { throw new RuntimeException(e); } this.indexLastBufSize = bytesRef.length;//cache heuristic return new Field[]{new BinaryDocValuesField(getFieldName(), bytesRef)}; }
@Override public Field[] createIndexableFields(Shape shape) { int bufSize = Math.max(128, (int) (this.indexLastBufSize * 1.5));//50% headroom over last ByteArrayOutputStream byteStream = new ByteArrayOutputStream(bufSize); final BytesRef bytesRef = new BytesRef();//receiver of byteStream's bytes try { ctx.getBinaryCodec().writeShape(new DataOutputStream(byteStream), shape); //this is a hack to avoid redundant byte array copying by byteStream.toByteArray() byteStream.writeTo(new FilterOutputStream(null/*not used*/) { @Override public void write(byte[] b, int off, int len) throws IOException { bytesRef.bytes = b; bytesRef.offset = off; bytesRef.length = len; } }); } catch (IOException e) { throw new RuntimeException(e); } this.indexLastBufSize = bytesRef.length;//cache heuristic return new Field[]{new BinaryDocValuesField(getFieldName(), bytesRef)}; }
/** * Build a lucene {@link Document} to be stored in the queryindex from a query entry * @param id the query id * @param mq the MonitorQuery to be indexed * @param query the (possibly partial after decomposition) query to be indexed * @return a Document that will be indexed in the Monitor's queryindex */ protected Document buildIndexableQuery(String id, MonitorQuery mq, QueryCacheEntry query) { Document doc = presearcher.indexQuery(query.matchQuery, mq.getMetadata()); doc.add(new StringField(FIELDS.id, id, Field.Store.NO)); doc.add(new StringField(FIELDS.del, id, Field.Store.NO)); doc.add(new SortedDocValuesField(FIELDS.id, new BytesRef(id))); doc.add(new BinaryDocValuesField(FIELDS.hash, query.hash)); if (storeQueries) doc.add(new BinaryDocValuesField(FIELDS.mq, MonitorQuery.serialize(mq))); return doc; }
@SneakyThrows public static void add(@NonNull Document doc, @NonNull String name, String value, FieldOptions options) { if (value != null) { if (options.isIndex()) { doc.add(createKeyword(name, value, options)); } if (options.isDocValue()) { doc.add(new BinaryDocValuesField(name, new BytesRef(value))); } if (!options.isIndex() && options.isStore()) { doc.add(new StoredField(name, value)); } } }
@Override public Void visit(Node node) { Document document = new Document(); document.add(new Field("class", "node", classField)); document.add(new BinaryDocValuesField("class_value", class_nodeByteRef)); document.add(new Field("node.identity", String.valueOf(node.getId()), classField)); document.add(new NumericDocValuesField("node.identity_value", node.getId())); if (node.isLoaded()) { document.add(new DoubleField("node.latitude", node.getLatitude(), coordinateDoubleField)); document.add(new DoubleField("node.longitude", node.getLongitude(), coordinateDoubleField)); } else if (log.isInfoEnabled()) { log.info("Indexing node " + node.getId() + " which has not been loaded. Coordinates will not be searchable."); } addObjectFields(node, document); try { indexWriter.updateDocument(new Term("node.identity", String.valueOf(node.getId())), document); } catch (IOException e) { throw new RuntimeException(e); } return null; }
@Override public IndexableField createField(SchemaField field, Object val /*, float boost*/) { if (val == null) return null; if (!field.stored()) { return null; } byte[] buf = null; int offset = 0, len = 0; if (val instanceof byte[]) { buf = (byte[]) val; len = buf.length; } else if (val instanceof ByteBuffer && ((ByteBuffer)val).hasArray()) { ByteBuffer byteBuf = (ByteBuffer) val; buf = byteBuf.array(); offset = byteBuf.position(); len = byteBuf.limit() - byteBuf.position(); } else { String strVal = val.toString(); //the string has to be a base64 encoded string buf = Base64.base64ToByteArray(strVal); offset = 0; len = buf.length; } Field f = new org.apache.lucene.document.BinaryDocValuesField(field.getName(), new BytesRef(buf, offset, len)); // Field f = new org.apache.lucene.document.StoredField(field.getName(), buf, offset, len); //f.setBoost(boost); return f; } }
@SneakyThrows public static void add(@NonNull Document doc, @NonNull String name, String value, FieldOptions options) { if (value != null) { if (options.isIndex()) { doc.add(new TextField(name, value, options.isStore() ? Field.Store.YES : Field.Store.NO)); } if (options.isDocValue()) { doc.add(new BinaryDocValuesField(name, new BytesRef(value))); } if (!options.isIndex() && options.isStore()) { doc.add(new StoredField(name, value)); } } }
@Override protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException { Field uid = new Field(NAME, Uid.createUid(context.stringBuilder(), context.type(), context.id()), Defaults.FIELD_TYPE); context.uid(uid); fields.add(uid); if (fieldType().hasDocValues()) { fields.add(new BinaryDocValuesField(NAME, new BytesRef(uid.stringValue()))); } }
doc.add(new BinaryDocValuesField(indexFieldName, new BytesRef(bytes, 0, upto)));
doc.add(new BinaryDocValuesField("json", new BytesRef(json.toString())));
doc.add(new BinaryDocValuesField(indexFieldName, new BytesRef(bytes, 0, upto)));
document.add(new BinaryDocValuesField("class_value", class_wayByteRef)); document.add(new Field("way.identity", String.valueOf(way.getId()), classField)); document.add(new NumericDocValuesField("way.identity_value", way.getId()));