logger.info("Started building the inverted index..."); if (currentIndex.getCollectionStatistics().getNumberOfUniqueTerms() == 0) if (currentIndex.getCollectionStatistics().getNumberOfDocuments() == 0)
if (currentIndex.getCollectionStatistics().getNumberOfUniqueTerms() == 0) if (currentIndex.getCollectionStatistics().getNumberOfDocuments() == 0)
IndexOnDisk src2 = Index.createIndex(index2[0], index2[1]); IndexOnDisk dst = Index.createNewIndex(outputIndex[0], outputIndex[1]); logger.info("Merging "+ src1+ " ("+src1.getCollectionStatistics().getNumberOfDocuments()+" docs) & " + src2 +" ("+src2.getCollectionStatistics().getNumberOfDocuments()+" docs) to " + dst); if (ApplicationSetup.BLOCK_INDEXING) sMerger = new BlockStructureMerger(src1, src2, dst); logger.debug("new index has " + dst.getCollectionStatistics().getNumberOfDocuments() + " docs"); try{ src1.close(); src2.close(); dst.close();
valueFactoryClass); TerrierTimer tt = new TerrierTimer("Recompressing inverted index", index.getCollectionStatistics().getNumberOfPointers()); tt.start(); try{
valueFactoryClass); TerrierTimer tt = new TerrierTimer("Recompressing inverted index", index.getCollectionStatistics().getNumberOfPointers()); tt.start(); try{
Iterator<String[]> metaIn = (Iterator<String[]>) index.getIndexStructureInputStream("meta"); int docid = 0; TerrierTimer tt = new TerrierTimer("Reading metaindex", index.getCollectionStatistics().getNumberOfDocuments()); tt.start(); try{
Iterator<String[]> metaIn = (Iterator<String[]>) index.getIndexStructureInputStream("meta"); int docid = 0; TerrierTimer tt = new TerrierTimer("Reading metaindex", index.getCollectionStatistics().getNumberOfDocuments()); tt.start(); try{
long numberOfPointers = 0; int numberOfUniqueTerms = index.getCollectionStatistics().getNumberOfUniqueTerms(); Iterator<Map.Entry<String, LexiconEntry>> lexiconStream = (Iterator<Map.Entry<String, LexiconEntry>>)this.index.getIndexStructureInputStream("lexicon");
long _numberOfTokens = 0; long _numberOfPointers = 0; int _numberOfUniqueTerms = index.getCollectionStatistics().getNumberOfUniqueTerms();
final long totalTokens = index.getCollectionStatistics().getNumberOfTokens(); final String iterationSuffix = (processTokens > totalTokens) ? " of 1 iteration" } while(firstDocid < index.getCollectionStatistics().getNumberOfDocuments()); assert firstDocid == index.getCollectionStatistics().getNumberOfDocuments() : " firstDocid=" + firstDocid; +index.getCollectionStatistics().getNumberOfTokens()+ ", found " + numberOfTokensFound);
int numberOfDocs1 = srcIndex1.getCollectionStatistics().getNumberOfDocuments(); int numberOfDocs2 = srcIndex2.getCollectionStatistics().getNumberOfDocuments(); ? (int)srcIndex1.getCollectionStatistics().getNumberOfUniqueTerms() : 0;
assert docid < this.index.getCollectionStatistics().getNumberOfDocuments(); assert termid < this.index.getCollectionStatistics().getNumberOfUniqueTerms();
int[] fieldFs = null; TerrierTimer tt = new TerrierTimer("Inverted index processing for this iteration", index.getCollectionStatistics().getNumberOfPointers()); tt.start(); try{
int[] fieldFs = null; TerrierTimer tt = new TerrierTimer("Inverted index processing for this iteration", index.getCollectionStatistics().getNumberOfPointers()); tt.start(); try{