/** Print the contents of the meta index */ @SuppressWarnings("unchecked") public static void printMetaIndex(Index index, String structureName) throws IOException { Iterator<String[]> inputStream = (Iterator<String[]>)index.getIndexStructureInputStream(structureName); while(inputStream.hasNext()) { System.out.println(ArrayUtils.join(inputStream.next(), ", ")); } IndexUtil.close(inputStream); }
/** Print the contents of the meta index */ @SuppressWarnings("unchecked") public static void printMetaIndex(Index index, String structureName) throws IOException { Iterator<String[]> inputStream = (Iterator<String[]>)index.getIndexStructureInputStream(structureName); while(inputStream.hasNext()) { System.out.println(ArrayUtils.join(inputStream.next(), ", ")); } IndexUtil.close(inputStream); }
/** Print the contents of the document index */ @SuppressWarnings("unchecked") public static void printDocumentIndex(Index index, String structureName) throws IOException { Iterator<DocumentIndexEntry> iterator = (Iterator<DocumentIndexEntry>)index.getIndexStructureInputStream(structureName); int docid =0; while(iterator.hasNext()) { DocumentIndexEntry die = iterator.next(); System.out.println(docid +": " + die.toString()); docid++; } close(iterator); }
/** * Prints Lexicon * @param index * @param structureName * @throws IOException */ @SuppressWarnings("unchecked") public static void printLexicon(Index index, String structureName) throws IOException { Iterator<Map.Entry<?,LexiconEntry>> lexiconStream = (Iterator<Map.Entry<?,LexiconEntry>>)index.getIndexStructureInputStream(structureName); while (lexiconStream.hasNext()) { Map.Entry<?, LexiconEntry> lee = lexiconStream.next(); System.out.println(lee.getKey().toString()+","+lee.getValue().toString()); } IndexUtil.close(lexiconStream); } }
/** Print the contents of the document index */ @SuppressWarnings("unchecked") public static void printDocumentIndex(Index index, String structureName) throws IOException { Iterator<DocumentIndexEntry> iterator = (Iterator<DocumentIndexEntry>)index.getIndexStructureInputStream(structureName); int docid =0; while(iterator.hasNext()) { DocumentIndexEntry die = iterator.next(); System.out.println(docid +": " + die.toString()); docid++; } close(iterator); }
/** * Prints Lexicon * @param index * @param structureName * @throws IOException */ @SuppressWarnings("unchecked") public static void printLexicon(Index index, String structureName) throws IOException { Iterator<Map.Entry<?,LexiconEntry>> lexiconStream = (Iterator<Map.Entry<?,LexiconEntry>>)index.getIndexStructureInputStream(structureName); while (lexiconStream.hasNext()) { Map.Entry<?, LexiconEntry> lee = lexiconStream.next(); System.out.println(lee.getKey().toString()+","+lee.getValue().toString()); } IndexUtil.close(lexiconStream); } }
BufferedReader br = Files.openFileReader(inputFile); staticScores = new double[index.getCollectionStatistics().getNumberOfDocuments()]; Iterator<String[]> metaIn = (Iterator<String[]>) index.getIndexStructureInputStream("meta");
BufferedReader br = Files.openFileReader(inputFile); staticScores = new double[index.getCollectionStatistics().getNumberOfDocuments()]; Iterator<String[]> metaIn = (Iterator<String[]>) index.getIndexStructureInputStream("meta");
@SuppressWarnings("unchecked") protected void checkStream(Index index, String name, String[] docnos, int ith) throws Exception { Iterator<String[]> metaIn = (Iterator<String[]>) index.getIndexStructureInputStream(name); assertNotNull(metaIn); int i = 0; while(metaIn.hasNext()) { String[] data = metaIn.next(); assertEquals(docnos[i], data[ith]); i++; } assertEquals(docnos.length, i); IndexUtil.close(metaIn); }
PostingIndexInputStream bpiis = (PostingIndexInputStream) index.getIndexStructureInputStream(structureName); bpiis.print(); bpiis.close(); Iterator<? extends Writable> in = (Iterator<? extends Writable>) index.getIndexStructureInputStream(structureName); while(in.hasNext())
PostingIndexInputStream bpiis = (PostingIndexInputStream) index.getIndexStructureInputStream(structureName); bpiis.print(); bpiis.close(); Iterator<? extends Writable> in = (Iterator<? extends Writable>) index.getIndexStructureInputStream(structureName); while(in.hasNext())
@SuppressWarnings("unchecked") @Test public void testScanForTokens() throws Exception { Index index = IndexTestUtils.makeIndex( new String[]{"doc1", "doc2"}, new String[]{"Quick fast brown fox", "jumped huge black lazy dog"});//no stopwords long[] testConstraint = new long[]{0, 1, 4, 5, 9, 10}; int[] numDocs = new int[]{1,1,1,2,2,2}; for(int i=0;i<testConstraint.length;i++) { Iterator<DocumentIndexEntry> dois = (Iterator<DocumentIndexEntry>) index.getIndexStructureInputStream("document"); assertEquals("Looking for numdocs to match " + testConstraint[i] + " tokens", numDocs[i], Inverted2DirectIndexBuilder.scanDocumentIndexForTokens(testConstraint[i], dois)); IndexUtil.close(dois); } }
@Override public void checkIndex(BatchEndToEndTest test, Index index) throws Exception { //no check correct type of all structures PostingIndexInputStream bpiis; IterablePosting ip; PostingIndex<Pointer> bpi; //check stream structures bpiis = (PostingIndexInputStream) index.getIndexStructureInputStream("direct"); ip = bpiis.next(); assertTrue(ip instanceof BlockPosting); bpiis.close(); bpiis = (PostingIndexInputStream) index.getIndexStructureInputStream("inverted"); ip = bpiis.next(); assertTrue(ip instanceof BlockPosting); bpiis.close(); //check random structures bpi = (PostingIndex<Pointer>) index.getInvertedIndex(); ip = bpi.getPostings(index.getLexicon().getLexiconEntry(0).getValue()); assertTrue(ip instanceof BlockPosting); bpi = (PostingIndex<Pointer>) index.getDirectIndex(); ip = bpi.getPostings(index.getDocumentIndex().getDocumentEntry(0)); assertTrue(ip instanceof BlockPosting); } }
protected void testInverted(Index index) throws Exception { PostingIndexInputStream piis = (PostingIndexInputStream) index.getIndexStructureInputStream("inverted"); IterablePosting ip = null; BlockPosting bp = null; while(piis.hasNext()) { ip = piis.getNextPostings(); if (ip == null) continue; bp = (BlockPosting) ip; while(ip.next() != IterablePosting.EOL) { int tf = bp.getFrequency(); int[] blocks = bp.getPositions(); assertTrue("blocks.length="+blocks.length + " tf="+tf, blocks.length <= tf); assertTrue("blocks.length="+blocks.length + " tf="+tf + ", blocks longer than max "+ maxBlocks, blocks.length <= maxBlocks); } } piis.close(); } }
Iterator<Map.Entry<String, LexiconEntry>> lexIn = (Iterator<Entry<String, LexiconEntry>>) index.getIndexStructureInputStream("lexicon"); assertNotNull(lexIn); int count = 0;
Iterator<DocumentIndexEntry> iDie = (Iterator<DocumentIndexEntry>) index.getIndexStructureInputStream("document"); assertNotNull("Failed to get a document inputstream", iDie); while(iDie.hasNext())
bpiis = (PostingIndexInputStream) index.getIndexStructureInputStream("direct"); ip = bpiis.next(); assertTrue(ip instanceof FieldPosting); bpiis.close(); bpiis = (PostingIndexInputStream) index.getIndexStructureInputStream("inverted"); ip = bpiis.next(); assertTrue(ip instanceof FieldPosting);
Iterator<String[]> iMi = (Iterator<String[]>) index.getIndexStructureInputStream("meta");
PostingIndexInputStream iiis = (PostingIndexInputStream) index.getIndexStructureInputStream("inverted"); assertNotNull(iiis); int ithTerm = -1;
final PostingIndexInputStream piis = (PostingIndexInputStream) index.getIndexStructureInputStream("direct"); assertNotNull("No direct index input stream found", piis); while(piis.hasNext())