MemoryIndex index = new MemoryIndex(withOffsets); for (Map.Entry<String, Collection<Object>> entry : values.entrySet()) { String field = entry.getKey();
public LuceneTokenCounter(Analyzer generalAnalyzer) throws IOException { memoryIndex = new MemoryIndex(); IndexSearcher searcher = memoryIndex.createSearcher(); leafReader = (LeafReader)searcher.getIndexReader(); this.generalAnalyzer = generalAnalyzer; }
/** * Builds a MemoryIndex from a lucene {@link Document} using an analyzer * @param document the document to index * @param analyzer the analyzer to use * @param storeOffsets <code>true</code> if offsets should be stored * @param storePayloads <code>true</code> if payloads should be stored * @param maxReusedBytes the number of bytes that should remain in the internal memory pools after {@link #reset()} is called * @return a MemoryIndex */ public static MemoryIndex fromDocument(Iterable<? extends IndexableField> document, Analyzer analyzer, boolean storeOffsets, boolean storePayloads, long maxReusedBytes) { MemoryIndex mi = new MemoryIndex(storeOffsets, storePayloads, maxReusedBytes); for (IndexableField field : document) { mi.addField(field, analyzer); } return mi; }
/** * Builds a MemoryIndex from a lucene {@link Document} using an analyzer * @param document the document to index * @param analyzer the analyzer to use * @param storeOffsets <code>true</code> if offsets should be stored * @param storePayloads <code>true</code> if payloads should be stored * @param maxReusedBytes the number of bytes that should remain in the internal memory pools after {@link #reset()} is called * @return a MemoryIndex */ public static MemoryIndex fromDocument(Document document, Analyzer analyzer, boolean storeOffsets, boolean storePayloads, long maxReusedBytes) { MemoryIndex mi = new MemoryIndex(storeOffsets, storePayloads, maxReusedBytes); for (IndexableField field : document) { mi.addField(field, analyzer); } return mi; }
private MemoryIndex getOrCreateMemoryIndex(OCommandContext ctx) { MemoryIndex memoryIndex = (MemoryIndex) ctx.getVariable(MEMORY_INDEX); if (memoryIndex == null) { memoryIndex = new MemoryIndex(); ctx.setVariable(MEMORY_INDEX, memoryIndex); } memoryIndex.reset(); return memoryIndex; }
public static MemoryIndex getOrCreateMemoryIndex(OCommandContext ctx) { MemoryIndex memoryIndex = (MemoryIndex) ctx.getVariable(MEMORY_INDEX); if (memoryIndex == null) { memoryIndex = new MemoryIndex(); ctx.setVariable(MEMORY_INDEX, memoryIndex); } memoryIndex.reset(); return memoryIndex; }
public class MemoryIndexTest { private static final String DATE_FIELD = "date"; MemoryIndex index = new MemoryIndex(); NumericTokenStream numericTokenStream = new NumericTokenStream(); @Before public void init() { index.addField(DATE_FIELD, numericTokenStream.setIntValue(20141116)); } @Test public void testRange() { Query query = NumericRangeQuery.newIntRange(DATE_FIELD, 20141115, 20141118, true, true); assertTrue(index.search(query) > 0); } }
MemoryIndex index = new MemoryIndex(withOffsets); for (Map.Entry<String, Collection<Object>> entry : values.entrySet()) { String field = entry.getKey();
MemoryIndex index = new MemoryIndex(withOffsets); for (Map.Entry<String, Collection<Object>> entry : values.entrySet()) { String field = entry.getKey();
MemoryIndex index = new MemoryIndex(withOffsets); for (Map.Entry<String, Collection<Object>> entry : values.entrySet()) { String field = entry.getKey();
private static HashSet<String> performSearch(Analyzer a) throws IOException { HashSet<String> results = new HashSet<>(); for (File file : getTestFiles()) { MemoryIndex memoryIndex = new MemoryIndex(true); final List<String> lines = Files.readAllLines(file.toPath(), Charset.forName("UTF-8")); memoryIndex.addField("title", lines.get(0), a); StringBuilder sb = new StringBuilder(); for (String line : lines) { sb.append(line); } memoryIndex.addField("content", sb.toString(), a); IndexSearcher searcher = memoryIndex.createSearcher(); ExistsCollector collector = new ExistsCollector(); searcher.search(new TermQuery(new Term("content", "אני")), collector); if (collector.exists()) { results.add(file.getName()); } } return results; }
public boolean isDeleted(Document document, Object key, OIdentifiable value) { boolean match = false; List<String> strings = deleted.get(value.getIdentity().toString()); if (strings != null) { MemoryIndex memoryIndex = new MemoryIndex(); for (String string : strings) { Query q = engine.deleteQuery(string, value); memoryIndex.reset(); for (IndexableField field : document.getFields()) { memoryIndex.addField(field.name(), field.stringValue(), new KeywordAnalyzer()); } match = match || (memoryIndex.search(q) > 0.0f); } return match; } return match; }
private Fields generateTermVectors(Collection<GetField> getFields, boolean withOffsets, @Nullable Map<String, String> perFieldAnalyzer, Set<String> fields) throws IOException { /* store document in memory index */ MemoryIndex index = new MemoryIndex(withOffsets); for (GetField getField : getFields) { String field = getField.getName(); if (fields.contains(field) == false) { // some fields are returned even when not asked for, eg. _timestamp continue; } Analyzer analyzer = getAnalyzerAtField(field, perFieldAnalyzer); for (Object text : getField.getValues()) { index.addField(field, text.toString(), analyzer); } } /* and read vectors from it */ return MultiFields.getFields(index.createSearcher().getIndexReader()); }
memoryIndex = new MemoryIndex(true);
@Override public Object evaluateRecord(OIdentifiable iRecord, ODocument iCurrentResult, OSQLFilterCondition iCondition, Object iLeft, Object iRight, OCommandContext iContext, final ODocumentSerializer serializer) { OLuceneFullTextIndex index = involvedIndex(iRecord, iCurrentResult, iCondition, iLeft, iRight); if (index == null) { throw new OCommandExecutionException("Cannot evaluate lucene condition without index configuration."); } MemoryIndex memoryIndex = (MemoryIndex) iContext.getVariable(MEMORY_INDEX); if (memoryIndex == null) { memoryIndex = new MemoryIndex(); iContext.setVariable(MEMORY_INDEX, memoryIndex); } memoryIndex.reset(); try { // In case of collection field evaluate the query with every item until matched if (iLeft instanceof List && index.isCollectionIndex()) { return matchCollectionIndex((List) iLeft, iRight, index, memoryIndex); } else { return matchField(iLeft, iRight, index, memoryIndex); } } catch (ParseException e) { OLogManager.instance().error(this, "error occurred while building query", e); } catch (IOException e) { OLogManager.instance().error(this, "error occurred while building memory index", e); } return null; }
@Override public Object evaluateRecord(OIdentifiable iRecord, ODocument iCurrentResult, OSQLFilterCondition iCondition, Object iLeft, Object iRight, OCommandContext iContext) { OLuceneFullTextIndex index = involvedIndex(iRecord, iCurrentResult, iCondition, iLeft, iRight); if (index == null) { throw new OCommandExecutionException("Cannot evaluate lucene condition without index configuration."); } MemoryIndex memoryIndex = (MemoryIndex) iContext.getVariable("_memoryIndex"); if (memoryIndex == null) { memoryIndex = new MemoryIndex(); iContext.setVariable("_memoryIndex", memoryIndex); } memoryIndex.reset(); Document doc = index.buildDocument(iLeft); for (IndexableField field : doc.getFields()) { memoryIndex.addField(field.name(), field.stringValue(), index.analyzer(field.name())); } Query query = null; try { query = index.buildQuery(iRight); } catch (Exception e) { throw new OCommandExecutionException("Error executing lucene query.", e); } return memoryIndex.search(query) > 0.0f; }
final MemoryIndex indexer = new MemoryIndex(true, usePayloads);//offsets and payloads if (cacheIt) { assert !cachedTokenStream;