public void close() { if (index != null) { index.setIndexProperty("num.Terms", ""+numberOfTerms); index.setIndexProperty("num.Tokens", ""+numberOfTokens); index.setIndexProperty("num.Pointers", ""+numberOfPointers); for(int fi = 0; fi < numFields; fi++) { index.setIndexProperty("num.field."+fi+".Tokens", ""+ tokensF[fi]); } } } }
public void performMultiWayMerge() throws IOException { super.performMultiWayMerge(); String structureName = "inverted"; currentIndex.setIndexProperty("index."+structureName+".blocks", String.valueOf(this.BLOCK_SIZE)); currentIndex.setIndexProperty("index."+structureName+".blocks.max", String.valueOf(this.MAX_BLOCKS)); currentIndex.flush(); } }
index.addIndexStructure(structureName, "org.terrier.structures.CompressingMetaIndex", "org.terrier.structures.IndexOnDisk,java.lang.String", "index,structureName"); index.addIndexStructureInputStream(structureName, "org.terrier.structures.CompressingMetaIndex$InputStream", "org.terrier.structures.IndexOnDisk,java.lang.String", "index,structureName"); index.setIndexProperty("index."+structureName+".entries", ""+entryCount); index.setIndexProperty("index."+structureName+".compression-level", ""+ZIP_COMPRESSION_LEVEL); index.setIndexProperty("index."+structureName+".key-names", ArrayUtils.join(keyNames, ",")); index.setIndexProperty("index."+structureName+".value-lengths", ArrayUtils.join(valueLensChars, ",")); index.setIndexProperty("index."+structureName+".entry-length", ""+entryLengthBytes); index.setIndexProperty("index."+structureName+".data-source", currentOffset > MAX_MB_IN_MEM_RETRIEVAL * (long)1024 * (long)1024 ? "file" : "fileinmem"); index.setIndexProperty("index."+structureName+".index-source", currentIndexOffset > MAX_INDEX_MB_IN_MEM_RETRIEVAL* (long)1024 * (long)1024 ? "file" : "fileinmem"); index.setIndexProperty("index."+structureName+".reverse-key-names", ArrayUtils.join(forwardKeyNames, ",")); index.flush();
setIndexProperty("index.terrier.version", ApplicationSetup.TERRIER_VERSION); setIndexProperty("index.created", "" + System.currentTimeMillis()); setIndexProperty("num.Documents", "0"); setIndexProperty("num.Terms", "0"); setIndexProperty("num.Tokens", "0"); setIndexProperty("num.Pointers", "0"); loadUpdatingStatistics(); dirtyProperties = true;
setIndexProperty("index.terrier.version", ApplicationSetup.TERRIER_VERSION); setIndexProperty("index.created", "" + System.currentTimeMillis()); setIndexProperty("num.Documents", "0"); setIndexProperty("num.Terms", "0"); setIndexProperty("num.Tokens", "0"); setIndexProperty("num.Pointers", "0"); loadUpdatingStatistics(); dirtyProperties = true;
/** * Closes the underlying file after finished processing the collections. */ public void finishedCollections() { final int maxDocsEncodedDocid = Integer.parseInt( ApplicationSetup.getProperty("indexing.max.encoded."+structureName+"index.docs","5000")); if (index != null) { if (structureName.equals("document")) index.setIndexProperty("num.Documents", ""+numberOfDocumentIndexEntries); index.addIndexStructure(structureName, numberOfDocumentIndexEntries > maxDocsEncodedDocid ? "org.terrier.structures.FSADocumentIndex" : "org.terrier.structures.FSADocumentIndexInMem", "org.terrier.structures.IndexOnDisk,java.lang.String", "index,structureName"); index.addIndexStructureInputStream(structureName, "org.terrier.structures.FSADocumentIndex$FSADocumentIndexIterator", "org.terrier.structures.IndexOnDisk,java.lang.String", "index,structureName"); } close(); } }
this.currentIndex.setIndexProperty("max.term.length", ApplicationSetup.getProperty("max.term.length", ""+20)); LexiconOutputStream<String> lexStream = new FSOMapFileLexiconOutputStream(this.currentIndex, "lexicon", (super.numFields > 0 ? FieldLexiconEntry.Factory.class : BasicLexiconEntry.Factory.class)); Files.delete(_fileNames[i][1]); currentIndex.setIndexProperty("num.Terms", ""+numberOfUniqueTerms); currentIndex.setIndexProperty("num.Pointers", ""+numberOfPointers); currentIndex.setIndexProperty("num.Tokens", ""+numberOfTokens); currentIndex.addIndexStructure( "inverted", ? fieldInvertedIndexPostingIteratorClass : basicInvertedIndexPostingIteratorClass )); currentIndex.setIndexProperty("index.inverted.fields.count", ""+FieldScore.FIELDS_COUNT ); currentIndex.setIndexProperty("index.inverted.fields.names", ArrayUtils.join(FieldScore.FIELD_NAMES, ",")); }catch(Exception e){ logger.error("Problem in performMultiWayMerge", e);
destIndex.setIndexProperty(property, srcIndex1.getIndexProperty(property, null));
index.setIndexProperty("index."+structureName+".termids", "aligned"); logger.info("All ids for structure "+structureName+ " are aligned, skipping " +FSOMapFileLexicon.ID_EXT+ " file"); dos.writeInt(indexof); dos.close(); index.setIndexProperty("index."+structureName+".termids", (numEntries > 15000000) ? "file" : "fileinmem"); oos.writeObject(map); oos.close(); index.setIndexProperty("index."+structureName+".bsearchshortcut", "charmap"); index.flush();
index.setIndexProperty("num.Terms", ""+numberOfUniqueTerms); index.setIndexProperty("num.Tokens", ""+numberOfTokens); index.setIndexProperty("num.Pointers", ""+numberOfPointers); System.gc();
index.setIndexProperty("num.Terms", ""+_numberOfUniqueTerms); index.setIndexProperty("num.Tokens", ""+_numberOfTokens); index.setIndexProperty("num.Pointers", ""+_numberOfPointers); index.flush(); System.gc();
"index.lexicon-valuefactory.parameter_types", "termpipelines"} ) destIndex.setIndexProperty(property, srcIndex1.getIndexProperty(property, null)); invOS.close(); destIndex.setIndexProperty("num.Documents", ""+numberOfDocuments); destIndex.addIndexStructure( "inverted", "index,structureName,lexicon-entry-inputstream,"+ compressionInvertedConfig.getPostingIteratorClass().getName()); destIndex.setIndexProperty("index.inverted.fields.count", ""+fieldCount); lexOutStream.close(); if (fieldCount > 0)
currentIndex.setIndexProperty("termpipelines", ApplicationSetup.getProperty("termpipelines", "Stopwords,PorterStemmer")); metaBuilder.close(); currentIndex.flush();
"index,structureName,document-inputstream,"+ (fieldCount > 0 ? fieldDirectIndexPostingIteratorClass : basicDirectIndexPostingIteratorClass)); index.setIndexProperty("index."+destinationStructure+".fields.count", ""+fieldCount ); index.setIndexProperty("index."+destinationStructure+".fields.names", index.getIndexProperty("index."+sourceStructure+".fields.names", ""));
); if (this.index.getIndexProperty("max.term.length", null) == null) this.index.setIndexProperty("max.term.length", ApplicationSetup.getProperty("max.term.length", ""+20)); this.index.addIndexStructure(defaultStructureName+"-valuefactory", lexiconEntryFactoryValueClass+"$Factory", valueFactoryParamTypes, valueFactoryParamValues); valueFactory = (FixedSizeWriteableFactory<LexiconEntry>)this.index.getIndexStructure(defaultStructureName+"-valuefactory");
currentIndex.setIndexProperty("num.Tokens", ""+numberOfTokens); currentIndex.setIndexProperty("termpipelines", ApplicationSetup.getProperty("termpipelines", "Stopwords,PorterStemmer")); if (FieldScore.FIELDS_COUNT > 0)
currentIndex.setIndexProperty("termpipelines", ApplicationSetup.getProperty("termpipelines", "Stopwords,PorterStemmer"));
destIndex.setIndexProperty(property, srcIndex1.getIndexProperty(property, null));