/** * Hook method that creates the right type of DocumentTree class. */ protected void createDocumentPostings(){ if (FieldScore.FIELDS_COUNT > 0) termsInDocument = new FieldDocumentPostingList(FieldScore.FIELDS_COUNT); else termsInDocument = new DocumentPostingList(); }
/** Insert a term into this document, occurs at given block id, and in the given field */ public void insert(String t, int fieldId, int blockId) { super.insert(t, fieldId); TIntHashSet blockids = null; if ((blockids = term_blocks.get(t)) == null) { term_blocks.put(t, blockids = new TIntHashSet(/*TODO */)); } blockids.add(blockId); blockCount++; }
/** {@inheritDoc} */ public void addTerms(DocumentPostingList docPostings, int docid) throws IOException{ for (String term : docPostings.termSet()) add(term, docid, docPostings.getFrequency(term), ((FieldDocumentPostingList)docPostings).getFieldFrequencies(term)); }
/** Insert a term into this document, occurs at given block id, and in the given fields */ public void insert(String t, int[] fieldIds, int blockId) { super.insert(t, fieldIds); TIntHashSet blockids = null; if ((blockids = term_blocks.get(t)) == null) { term_blocks.put(t, blockids = new TIntHashSet(/*TODO */)); } blockids.add(blockId); blockCount++; }
/** FIXME */ void createDocumentPostings() { if (numFields>0) termsInDocument = new FieldDocumentPostingList(numFields); else termsInDocument = new DocumentPostingList(); }
MemoryFieldsLexiconEntry le = new MemoryFieldsLexiconEntry(1, docContents.getFrequency(term), ((FieldDocumentPostingList)docContents).getFieldFrequencies(term)); ((MemoryFieldsInvertedIndex) inverted).add(termid, stats.getNumberOfDocuments(), docContents.getFrequency(term), ((FieldDocumentPostingList)docContents).getFieldFrequencies(term)); int[] ffreq = ((FieldDocumentPostingList)docContents).getFieldFrequencies(term); for (int i = 0; i < fieldcounts.length; i++) fieldcounts[i] += ffreq[i];
/** Insert a term into this document tf times, occurs at given block id, and in the given fields */ public void insert(int tf, String t, int[] fieldIds, int blockId) { super.insert(tf, t, fieldIds); TIntHashSet blockids = null; if ((blockids = term_blocks.get(t)) == null) { term_blocks.put(t, blockids = new TIntHashSet(/*TODO */)); } blockids.add(blockId); blockCount++; }
/** {@inheritDoc} */ @Override public void indexDocument(Document doc) throws Exception { if (doc == null) return; // Process terms through term pipeline. fdpl = new FieldDocumentPostingList(fieldtags.length); while (!doc.endOfDocument()) { String term = doc.getNextTerm(); if (term == null || term.equals("")) continue; docFields = doc.getFields(); pipeline.processTerm(term); } indexDocument(doc.getAllProperties(), fdpl); }
public void processTerm(String term) { if (term != null) { TIntHashSet freqs = new TIntHashSet(0); for (String docField : docFields) freqs.add(fieldIDs.get(docField)); if (fieldIDs.containsKey("ELSE") && freqs.size() == 0) freqs.add(fieldIDs.get("ELSE")); ((FieldDocumentPostingList) fdpl).insert(term, freqs.toArray()); } }
public void processTerm(String term) { /* null means the term has been filtered out (eg stopwords) */ if (term != null) { /* add term to Document tree */ for (String fieldName: termFields) { int tmp = fieldNames.get(fieldName); if (tmp > 0) { fields.add(tmp -1); } } if (ELSE_ENABLED && fields.size() == 0) { fields.add(ELSE_FIELD_ID); } ((FieldDocumentPostingList)termsInDocument).insert(term,fields.toArray()); numOfTokensInDocument++; fields.clear(); } }
public void processTerm(String term) { /* null means the term has been filtered out (eg stopwords) */ if (term != null) { /* add term to Document tree */ for (String fieldName: termFields) { int tmp = fieldNames.get(fieldName); if (tmp > 0) { fields.add(tmp -1); } } if (ELSE_ENABLED && fields.size() == 0) { fields.add(ELSE_FIELD_ID); } ((FieldDocumentPostingList)termsInDocument).insert(term,fields.toArray()); numOfTokensInDocument++; fields.clear(); } } @Override