Refine search
private void removeFromCache( long entityId ) throws IOException { IndexSearcher searcher = searcherManager.acquire(); try { Query query = type.idTermQuery( entityId ); TopDocs docs = searcher.search( query, 1 ); if ( docs.totalHits > 0 ) { Document document = searcher.doc( docs.scoreDocs[0].doc ); for ( IndexableField field : document.getFields() ) { String key = field.name(); Object value = field.stringValue(); removeFromCache( entityId, key, value ); } } } finally { searcherManager.release( searcher ); } }
try { TopScoreDocCollector collector = TopScoreDocCollector.create(10, true); searcher.search(query, collector); ScoreDoc[] docs = collector.topDocs().scoreDocs; for (int i = 0; i < docs.length; i++) { LuceneResult lr = new LuceneResult(); Document doc = searcher.doc(docs[i].doc); lr.lat = doc.getField("lat").numericValue().doubleValue(); lr.lng = doc.getField("lon").numericValue().doubleValue(); String category = doc.getField("category").stringValue().toLowerCase(); String code; if (doc.getField("code") != null){ code = "(" + doc.getField("code").stringValue() + ")"; } else { code = ""; if (doc.getField("category").stringValue().equals(Category.STOP.name()) || doc.getField("category").stringValue().equals(Category.CLUSTER.name())) { lr.id = doc.getField("id").stringValue(); String name = doc.getField("name").stringValue(); lr.description = category + " " + name + " " + code; result.add(lr);
IndexSearcher searcher = new IndexSearcher(reader); Query q; try { TopDocs top = searcher.search(q, n); Document doc = searcher.doc(top.scoreDocs[i].doc); IndexableField objser = doc.getField(QueryBuilder.OBJSER); int objver = readObjectVersion(doc); if (objver != INDEX_ANALYSIS_SETTINGS_OBJVER) { objser.binaryValue().bytes); } catch (ClassNotFoundException ex) {
private Object getFieldValue( IndexableField field ) { Number numericFieldValue = field.numericValue(); return numericFieldValue != null ? numericFieldValue : field.stringValue(); }
/** * Extract the Apache Geode key term from a lucene document */ public static Term getKeyTerm(Document doc) { IndexableField field = doc.getField(KEY_FIELD); if (field.stringValue() != null) { return new Term(KEY_FIELD, field.stringValue()); } else { return new Term(KEY_FIELD, field.binaryValue()); } }
Document doc = new Document(); doc.add(new Field(QueryBuilder.FULLPATH, path, string_ft_nstored_nanalyzed_norms)); StringWriter xrefOut = new StringWriter(); analyzer.analyze(doc, getStreamSource(path), xrefOut); Definitions definitions = Definitions.deserialize(doc.getField(QueryBuilder.TAGS).binaryValue().bytes);
protected void init() { // put all respective features into an in-memory cache ... if (isCaching && reader != null) { Bits liveDocs = MultiFields.getLiveDocs(reader); int docs = reader.numDocs(); featureCache = new LinkedHashMap<Integer, byte[]>(docs); try { Document d; for (int i = 0; i < docs; i++) { if (!(reader.hasDeletions() && !liveDocs.get(i))) { d = reader.document(i); if (d.getField(fieldName) != null) { cachedInstance.setByteArrayRepresentation(d.getField(fieldName).binaryValue().bytes, d.getField(fieldName).binaryValue().offset, d.getField(fieldName).binaryValue().length); // featureCache.put(i, new SearchItem(cachedInstance.getByteArrayRepresentation(), new SimpleResult(-1d, i, d.getValues(DocumentBuilder.FIELD_NAME_IDENTIFIER)[0]))); // featureCache.put(i, new SearchItem(i, cachedInstance.getByteArrayRepresentation(), d.getValues(DocumentBuilder.FIELD_NAME_IDENTIFIER)[0])); featureCache.put(i, cachedInstance.getByteArrayRepresentation()); } } } } catch (IOException e) { e.printStackTrace(); } } }
IndexSearcher searcher = new IndexSearcher(ireader); TopDocs top = searcher.search(q, 1); if (top.totalHits == 0) { Document doc = searcher.doc(top.scoreDocs[0].doc); String foundPath = doc.get(QueryBuilder.PATH); IndexableField tags = doc.getField(QueryBuilder.TAGS); if (tags != null) { return Definitions.deserialize(tags.binaryValue().bytes); ireader.close();
if (getBooleanParameter(req, "rewrite")) { final Query rewritten_q = q.rewrite(searcher .getIndexReader()); queryRow.put("rewritten_q", rewritten_q.toString()); final int freq = searcher.getIndexReader().docFreq((Term) term); freqs.put(term.toString(), freq); td = searcher.search(q, skip + limit); } else { td = searcher.search(q, skip + limit, sort); for (final IndexableField f : doc.getFields()) { if (!f.fieldType().stored()) { continue; final String name = f.name(); final Object value; if (f.numericValue() != null) { value = f.numericValue(); } else { value = f.stringValue(); fetch_ids[i - skip] = doc.get("_id");
private List<Pair> getTopXNeighbors(float[] originalVector, Integer limit) { IndexSearcher indexSearcher = getIndexSearcher(); IndexReader indexReader = indexSearcher.getIndexReader(); long now = System.currentTimeMillis(); FixedSizeOrderedList coll = new FixedSizeOrderedList(limit); CosineSimilarity cosineSimilarity = new CosineSimilarity(); try { Query query = new MatchAllDocsQuery(); TopDocs searchResult = indexSearcher.search(query, indexReader.numDocs()); for (ScoreDoc scoreDoc : searchResult.scoreDocs) { Document hitDoc = indexSearcher.doc(scoreDoc.doc); String word = hitDoc.getField(Word2VecIndexCreator.WORD_FIELD).stringValue(); coll.add(new ComparablePair(word, cosineSimilarity.cosineSimilarity(originalVector, getVector(hitDoc)))); } } catch (Exception e) { throw new RuntimeException(e); } LOG.debug("Computed nearest neighbors in " + (System.currentTimeMillis() - now)); return coll; }
@Test public void testWriteFields() { String[] fields = new String[] {"s", "i"}; PdxLuceneSerializer mapper = new PdxLuceneSerializer(); PdxInstance pdxInstance = mock(PdxInstance.class); when(pdxInstance.hasField("s")).thenReturn(true); when(pdxInstance.hasField("i")).thenReturn(true); when(pdxInstance.getField("s")).thenReturn("a"); when(pdxInstance.getField("i")).thenReturn(5); Document doc = invokeSerializer(mapper, pdxInstance, fields); assertEquals(2, doc.getFields().size()); assertEquals("a", doc.getField("s").stringValue()); assertEquals(5, doc.getField("i").numericValue()); }
long getMaxEventId(final String partitionName) { final List<File> allDirectories = getDirectoryManager().getDirectories(0L, Long.MAX_VALUE, partitionName); if (allDirectories.isEmpty()) { return -1L; } Collections.sort(allDirectories, DirectoryUtils.NEWEST_INDEX_FIRST); for (final File directory : allDirectories) { final EventIndexSearcher searcher; try { searcher = indexManager.borrowIndexSearcher(directory); } catch (final IOException ioe) { logger.warn("Unable to read from Index Directory {}. Will assume that the index is incomplete and not consider this index when determining max event ID", directory); continue; } try { final IndexReader reader = searcher.getIndexSearcher().getIndexReader(); final int maxDocId = reader.maxDoc() - 1; final Document document = reader.document(maxDocId); final long eventId = document.getField(SearchableFields.Identifier.getSearchableFieldName()).numericValue().longValue(); logger.info("Determined that Max Event ID indexed for Partition {} is approximately {} based on index {}", partitionName, eventId, directory); return eventId; } catch (final IOException ioe) { logger.warn("Unable to search Index Directory {}. Will assume that the index is incomplete and not consider this index when determining max event ID", directory, ioe); } finally { indexManager.returnIndexSearcher(searcher); } } return -1L; }
@Test public void testIgnoreInvalid() { String[] fields = new String[] {"s", "o", "s2"}; ReflectionLuceneSerializer mapper = new ReflectionLuceneSerializer(Type2.class, fields); Type2 type2 = new Type2("a", 1, 2L, 3.0, 4.0f, "b"); Document doc = invokeSerializer(mapper, type2, fields); assertEquals(2, doc.getFields().size()); assertEquals("a", doc.getField("s").stringValue()); assertEquals("b", doc.getField("s2").stringValue()); }
@Override public int compare(final Document o1, final Document o2) { final String filename1 = o1.get(FieldNames.STORAGE_FILENAME); final String filename2 = o2.get(FieldNames.STORAGE_FILENAME); final int filenameComp = filename1.compareTo(filename2); if (filenameComp != 0) { return filenameComp; } final IndexableField fileOffset1 = o1.getField(FieldNames.BLOCK_INDEX); final IndexableField fileOffset2 = o1.getField(FieldNames.BLOCK_INDEX); if ( fileOffset1 != null && fileOffset2 != null ) { final int blockIndexResult = Long.compare(fileOffset1.numericValue().longValue(), fileOffset2.numericValue().longValue()); if ( blockIndexResult != 0 ) { return blockIndexResult; } final long eventId1 = o1.getField(SearchableFields.Identifier.getSearchableFieldName()).numericValue().longValue(); final long eventId2 = o2.getField(SearchableFields.Identifier.getSearchableFieldName()).numericValue().longValue(); return Long.compare(eventId1, eventId2); } final long offset1 = o1.getField(FieldNames.STORAGE_FILE_OFFSET).numericValue().longValue(); final long offset2 = o2.getField(FieldNames.STORAGE_FILE_OFFSET).numericValue().longValue(); return Long.compare(offset1, offset2); } });
public void loadNN() { try { IndexSearcher indexSearcher = getIndexSearcher(); IndexReader indexReader = indexSearcher.getIndexReader(); for (int i = 0; i < indexReader.maxDoc(); ++i) { Document hitDoc = indexReader.document(i); String word = hitDoc.getField(Word2VecIndexCreator.WORD_FIELD).stringValue(); StoredField binaryVector = (StoredField) hitDoc.getField(Word2VecIndexCreator.VECTOR_FIELD); inMemoryNN.put(word, TypeConverter.toFloatArray(binaryVector.binaryValue().bytes)); } } catch (Exception e) { throw new RuntimeException(e); } }
public Map<String, String> getObjectHashes() throws RcIrException, IOException { Query query = new TermQuery(new Term(RcIrStaticField.DOCUMENT_TYPE.name(), RcIrDocumentType.OBJECT_HASH.name())); TopDocs topDocs = this.getIndexSearcher().search(query, Integer.MAX_VALUE); Map<String, String> objHashMap = new HashMap<String, String>(); Set<String> fieldsToLoad = new HashSet<String>(); fieldsToLoad.add(RcIrStaticField.EXT_OBJ_ID.getFieldName()); fieldsToLoad.add(RcIrStaticField.OBJECT_HASH.getFieldName()); int docId; Document doc; IndexableField extObjIdField; String extObjId; IndexableField objHashField; String objHash; for (int i = 0; i < topDocs.scoreDocs.length; i++) { docId = topDocs.scoreDocs[i].doc; doc = this.getIndexReader().document(docId, fieldsToLoad); extObjIdField = doc.getField(RcIrStaticField.EXT_OBJ_ID.getFieldName()); extObjId = extObjIdField.binaryValue().utf8ToString(); objHashField = doc.getField(RcIrStaticField.OBJECT_HASH.getFieldName()); objHash = objHashField.binaryValue().utf8ToString(); objHashMap.put(extObjId, objHash); } return objHashMap; }
private Map<String, String> toMap( Document d ) { final HashMap<String, String> result = new HashMap<String, String>(); for ( Object o : d.getFields() ) { IndexableField f = (IndexableField) o; if ( f.fieldType().stored()) { result.put( f.name(), f.stringValue() ); } } return result; }
@Test public void shouldIndexPrimitiveNumberIfRequested() { HeterogeneousLuceneSerializer mapper = new HeterogeneousLuceneSerializer(); Document doc = SerializerTestHelper.invokeSerializer(mapper, 53, new String[] {LuceneService.REGION_VALUE_FIELD}); assertEquals(1, doc.getFields().size()); assertEquals(53, doc.getField(LuceneService.REGION_VALUE_FIELD).numericValue()); }
/** * Retrieves the local ID for a specified lucene document, * within a given language. * * @param docId * @param language * @return */ public int getLocalIdFromDocId(int docId, Language language) { try { if (docId != -1) { Document document = searchers.get(language).doc(docId); return (Integer) document.getField(LuceneOptions.LOCAL_ID_FIELD_NAME).numericValue(); } else { LOG.warn("This docId does not exist: " + docId); return -1; } } catch (IOException e) { throw new RuntimeException(e); } }