IndexOnDisk indexSrc1 = Index.createIndex(args[0], args[1]); IndexOnDisk indexSrc2 = Index.createIndex(args[2], args[3]); IndexOnDisk indexDest = Index.createNewIndex(args[4], args[5]);
IndexOnDisk indexD = Index.createNewIndex(index.path, index.prefix + "-" + index.prefixID);
IndexOnDisk indexDest = Index.createNewIndex(args[4], args[5]);
private int merge(int partition1, int partition2) { // Source index 1. IndexOnDisk src1 = Index.createIndex(index.path, index.prefix + "-" + partition1); // Source index 2. IndexOnDisk src2 = Index.createIndex(index.path, index.prefix + "-" + partition2); // Destination index. IndexOnDisk indexD = Index.createNewIndex(index.path, index.prefix + "-" + index.prefixID); // Merge the index structures. StructureMerger merger = new StructureMerger(src1, src2, indexD); merger.mergeStructures(); logger.info("***REALTIME*** IncrementalIndex merged: " + partition1 + " and " + partition2 + " into " + index.prefixID); // Increment prefix ID. index.prefixID++; // Update list of indices. merged.add(partition1); merged.add(partition2); purgeMerged(); synchronized (indices) { // FIXME } // Return prefixID of new partition. return index.prefixID - 1; }
IndexOnDisk indexSrc1 = Index.createIndex(args[0], args[1]); IndexOnDisk indexSrc2 = Index.createIndex(args[2], args[3]); IndexOnDisk indexDest = Index.createNewIndex(args[4], args[5]);
@Test public void testCopyStructureDifferentIndex() throws Exception { Index i1 = Index.createNewIndex(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX + "i1"); Index i2 = Index.createNewIndex(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX + "i2"); i1.addIndexStructure("test1", Object.class.getName(), "", ""); IndexUtil.copyStructure(i1, i2, "test1", "test2"); assertEquals(Object.class, i2.getIndexStructure("test2").getClass()); }
IndexOnDisk newIndex = Index.createNewIndex(path, prefix+"-tmp");
@Test public void testCopyStructureSameIndex() throws Exception { Index i1 = Index.createNewIndex(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX + "i1"); i1.addIndexStructure("test1", Object.class.getName(), "", ""); IndexUtil.copyStructure(i1, i1, "test1", "test2"); assertEquals(Object.class, i1.getIndexStructure("test2").getClass()); }
protected Matching getMatching(String[] rows, String[] docnos) throws Exception { File tmpFile = folder.newFile("tmp.res"); Writer w = Files.writeFileWriter(tmpFile); for(String row : rows) w.append(row + "\n"); w.close(); Index index = Index.createNewIndex(folder.newFolder("index").toString(), "data"); index.setIndexProperty("num.Documents", ""+docnos.length); IndexUtil.forceStructure(index, "meta", new ArrayMetaIndex(docnos)); Matching rtr = new TRECResultsMatching(index, tmpFile.toString()); return rtr; }
numberOfTokens = numberOfPointers = 0; createMemoryPostings(); currentIndex = Index.createNewIndex(path, prefix); docIndexBuilder = new DocumentIndexBuilder(currentIndex, "document"); metaBuilder = createMetaIndexBuilder();
@Test public void testKeysSubsetConfigurationMismatch() throws IOException { exception.expect(IllegalArgumentException.class); CompressingMetaIndexBuilder x = new CompressingMetaIndexBuilder( Index.createNewIndex(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX), new String[]{"docno"}, new int[]{20}, new String[]{"url"}); x.close(); }
numberOfTokens = numberOfPointers = 0; createMemoryPostings(); currentIndex = Index.createNewIndex(path, prefix); docIndexBuilder = new DocumentIndexBuilder(currentIndex, "document"); metaBuilder = createMetaIndexBuilder();
IndexOnDisk index = Index.createNewIndex(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX);
IndexOnDisk src1 = Index.createIndex(index1[0], index1[1]); IndexOnDisk src2 = Index.createIndex(index2[0], index2[1]); IndexOnDisk dst = Index.createNewIndex(outputIndex[0], outputIndex[1]); logger.info("Merging "+ src1+ " ("+src1.getCollectionStatistics().getNumberOfDocuments()+" docs) & " + src2 +" ("+src2.getCollectionStatistics().getNumberOfDocuments()+" docs) to " + dst);
@Test(expected=IllegalArgumentException.class) public void testNullTermException() throws Exception { IndexOnDisk index = Index.createNewIndex( ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX); LexiconBuilder lb = new LexiconBuilder(index, "lexicon", new TermCodes()); lb.addTerm("", 0); }
IndexOnDisk newIndex = Index.createNewIndex(path, prefix);
Index createLexiconIndex(String[] tokens) throws Exception { IndexOnDisk index = Index.createNewIndex( ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX); LexiconBuilder lb = new LexiconBuilder(index, "lexicon", new TermCodes()); for(String tok: tokens) { lb.addTerm(tok, 1); } lb.finishedDirectIndexBuild(); lb.finishedInvertedIndexBuild(); return index; }
public Index write(String path, String prefix) throws IOException { IndexOnDisk index = Index.createNewIndex(path, prefix);
protected void testBase(String name, String[] keyNames, int[] keyLengths, String[] revKeys, String[][] data) throws Exception IndexOnDisk index = Index.createNewIndex(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX); assertNotNull("Index should not be null", index); MetaIndexBuilder b = new CompressingMetaIndexBuilder(index, name,
@SuppressWarnings("unchecked") @Test public void testSimple() throws IOException { Index index = Index.createNewIndex(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX); CompressionConfiguration cc = getConfig("inverted", new String[0], 0,0); AbstractPostingOutputStream pos = cc.getPostingOutputStream(((IndexOnDisk)index).getPath() + "/" + ((IndexOnDisk)index).getPrefix() + ".inverted" + cc.getStructureFileExtension()); Pointer p = pos.writePostings(new ArrayOfBasicIterablePosting(new int[]{0, 1}, new int[]{1,2})); pos.close(); cc.writeIndexProperties(index, "lexicon-entry-inputstream"); index.flush(); assertTrue(index.hasIndexStructure("inverted")); PostingIndex<Pointer> inv = (PostingIndex<Pointer>) index.getIndexStructure("inverted"); IterablePosting ip = inv.getPostings(p); assertNotNull(ip); assertEquals(0, ip.next()); assertEquals(1, ip.getFrequency()); assertEquals(1, ip.next()); assertEquals(2, ip.getFrequency()); index.close(); IndexUtil.deleteIndex(ApplicationSetup.TERRIER_INDEX_PATH, ApplicationSetup.TERRIER_INDEX_PREFIX); } }