if (ArrayUtils.contains(results.getMetaKeys(), metaIndexDocumentKey)) docnos = results.stream().map(doc -> doc.getMetadata(metaIndexDocumentKey)).toArray(size -> new String[size]); } else if (index == null && q instanceof Request) { final MetaIndex metaIndex = ((Request)q).getIndex().getMetaIndex(); docnos = metaIndex.getItems(metaIndexDocumentKey, results.stream().mapToInt(doc -> doc.getDocid()).toArray()); } else if (index != null) { docnos = index.getMetaIndex().getItems(metaIndexDocumentKey, results.stream().mapToInt(doc -> doc.getDocid()).toArray()); } else { throw new IllegalStateException("No source for " +metaIndexDocumentKey+ " found. Perhaps you need to set control decorate:on"); if (doc.getScore() == Double.NEGATIVE_INFINITY) continue; sbuffer.append(queryIdExpanded); sbuffer.append(i); sbuffer.append(" "); sbuffer.append(doc.getScore()); sbuffer.append(methodExpanded); if (i % limit == 0) {
line = line.trim(); String[] parts = line.split("\\s+", 6); rtr.add(new ScoredDoc(0, Double.parseDouble(parts[4]), (short) 0, new String[] { parts[2] }, metaOffset));
if (Double.isInfinite(doc.getScore()) && doc.getScore()< 0) continue; sbuffer.append(i); sbuffer.append(" "); sbuffer.append(ArrayUtils.join(doc.getAllMetadata(), ' ')); sbuffer.append(" "); if (printDocid) sbuffer.append(doc.getDocid()); sbuffer.append(" "); sbuffer.append(doc.getScore()); sbuffer.append('\n');
@Test public void testOneDocumentTwoKeys() throws Exception { Index index = createIndexAbstract(); Manager m = new LocalManager(index); SearchRequest srq = m.newSearchRequest("test", "fox"); srq.setControl(SearchRequest.CONTROL_WMODEL, TF_IDF.class.getName()); m.runSearchRequest(srq); ResultSet rs = ((Request) srq).getResultSet(); assertEquals(1, rs.getResultSize()); SimpleDecorate decorate = new SimpleDecorate(); decorate.new_query(m, srq, rs); decorate.filter(m, srq, rs, 0, rs.getDocids()[0]); assertEquals(2, rs.getMetaKeys().length); assertEquals("doc1", rs.getMetaItems("filename")[0]); assertEquals("The quick brown fox jumps over the lazy dog", StringTools.escape(ESCAPE.JAVA, rs.getMetaItems("abstract")[0])); ScoredDocList res2 = srq.getResults(); assertEquals(1, res2.size()); ScoredDoc doc1 = res2.get(0); assertEquals(0, doc1.getDocid()); assertEquals("doc1", doc1.getMetadata("filename")); assertEquals("The quick brown fox jumps over the lazy dog", doc1.getMetadata("abstract")); }
@Test public void testRequirementsFields() throws Exception { ApplicationSetup.setProperty("FieldTags.process", "title,content"); Index index = IndexTestUtils.makeIndexFields( new String[]{"doc1", "doc2"}, new String[]{ "<title>Animal</title><content>The quick brown fox jumps over the lazy dog</content>", //doc1 "<title>Dog</title><content>how much is that animal in the window</content>"}); //doc2 Manager m = new LocalManager(index); SearchRequest srq = null; //4, are documents retrieved: two terms, one of which is negative requirement srq = m.newSearchRequest("test1", "dog -title:Animal"); srq.setControl(SearchRequest.CONTROL_WMODEL, PL2.class.getName()); srq.setControl(SearchRequest.CONTROL_MATCHING, getMatchingClass().getName()); m.runSearchRequest(srq); //System.err.println(srq.getResultSet().getResultSize()); /*for (int i =0; i<srq.getResultSet().getDocids().length; i++) { System.err.println(" "+srq.getResultSet().getDocids()[i]+" "+srq.getResultSet().getScores()[i]); }*/ assertEquals(1, srq.getResults().size()); assertEquals(1, srq.getResults().get(0).getDocid()); }
@Test public void itWorksBlocksFeatures() throws Exception { Index index = IndexTestUtils.makeIndexBlocks(new String[]{"doc1"}, new String[]{"token1 token2 token3"}); // String path = ((IndexOnDisk)index).getPath(); // String prefix = ((IndexOnDisk)index).getPrefix(); int port = new Random().nextInt(65536-1024)+1024; System.err.println("itWorksBlocksFeatures: Index is " + index.getIndexRef().toString()); String uri = "http://127.0.0.1:"+port+"/"; HttpServer server = makeServer(index, uri); SearchResource.reinit(); index.close(); ApplicationSetup.setProperty("fat.featured.scoring.matching.features", "WMODEL:BM25;WMODEL:PL2;DSM:"+DFRDependenceScoreModifier.class.getSimpleName()); Manager restManager = ManagerFactory.from(IndexRef.of(uri)); assertNotNull(restManager); SearchRequest srq = restManager.newSearchRequestFromQuery("\"token1 token2\""); srq.setControl(SearchRequest.CONTROL_MATCHING, FatFeaturedScoringMatching.class.getName() + "," + FatFull.class.getName() ); restManager.runSearchRequest(srq); assertEquals(1, srq.getResults().size()); assertEquals("doc1", srq.getResults().get(0).getMetadata("docno")); server.shutdown().get(); //IndexUtil.deleteIndex(path, prefix); }
@Test public void itWorks() throws Exception { Index index = IndexTestUtils.makeIndex(new String[]{"doc1"}, new String[]{"token1 token2 token3"}); // String path = ((IndexOnDisk)index).getPath(); // String prefix = ((IndexOnDisk)index).getPrefix(); int port = new Random().nextInt(65536-1024)+1024; String uri = "http://127.0.0.1:"+port+"/"; System.err.println("itWorks: Index is " + index.getIndexRef().toString()); HttpServer server = makeServer(index, uri); SearchResource.reinit(); index.close(); Manager restManager = ManagerFactory.from(IndexRef.of(uri)); assertNotNull(restManager); SearchRequest srq = restManager.newSearchRequestFromQuery("token1"); restManager.runSearchRequest(srq); assertEquals(1, srq.getResults().size()); assertEquals("doc1", srq.getResults().get(0).getMetadata("docno")); server.shutdown().get(); //IndexUtil.deleteIndex(path, prefix); }
if (ArrayUtils.contains(results.getMetaKeys(), metaIndexDocumentKey)) docnos = results.stream().map(doc -> doc.getMetadata(metaIndexDocumentKey)).toArray(size -> new String[size]); } else if (index == null && q instanceof Request) { final MetaIndex metaIndex = ((Request)q).getIndex().getMetaIndex(); docnos = metaIndex.getItems(metaIndexDocumentKey, results.stream().mapToInt(doc -> doc.getDocid()).toArray()); } else if (index != null) { docnos = index.getMetaIndex().getItems(metaIndexDocumentKey, results.stream().mapToInt(doc -> doc.getDocid()).toArray()); } else { throw new IllegalStateException("No source for " +metaIndexDocumentKey+ " found. Perhaps you need to set control decorate:on"); if (doc.getScore() == Double.NEGATIVE_INFINITY) continue; sbuffer.append(queryIdExpanded); sbuffer.append(i); sbuffer.append(" "); sbuffer.append(doc.getScore()); sbuffer.append(methodExpanded); if (i % limit == 0) {
if (Double.isInfinite(doc.getScore()) && doc.getScore()< 0) continue; sbuffer.append(i); sbuffer.append(" "); sbuffer.append(ArrayUtils.join(doc.getAllMetadata(), ' ')); sbuffer.append(" "); if (printDocid) sbuffer.append(doc.getDocid()); sbuffer.append(" "); sbuffer.append(doc.getScore()); sbuffer.append('\n');
line = line.trim(); String[] parts = line.split("\\s+", 6); rtr.add(new ScoredDoc(0, Double.parseDouble(parts[4]), (short) 0, new String[] { parts[2] }, metaOffset));
for(ScoredDoc doc : results) if (doc.getScore() == Double.NEGATIVE_INFINITY) continue; JSONObject result = new JSONObject(); result.put("rank", rank); result.put("docid", doc.getDocid()); result.put("score", doc.getScore()); for(String meta : metakeys) result.put(meta, doc.getMetadata(meta));
public ScoredDocList getResults() { ResultSet rs = this.getResultSet(); if (rs == null) throw new IllegalStateException("No resultset found"); final String[] metaKeys = rs.getMetaKeys(); Results rtr = new Results(metaKeys); Map<String,Integer> metaoffsets = new HashMap<>(); final int metaCount = metaKeys.length; if (metaKeys != null) for(int j=0;j<metaCount;j++) { metaoffsets.put(metaKeys[j], j); } final int size = rs.getResultSize(); final int[] docids = rs.getDocids(); final double[] scores = rs.getScores(); final short[] occ = rs.getOccurrences(); final String[][] meta = rs.allMetaItems(); for(int i=0;i<size;i++) { String[] metaValues = new String[metaCount]; for(int j=0;j<metaCount;j++) metaValues[j] = meta[j][i]; rtr.add(new ScoredDoc(docids[i], scores[i], occ[i], metaValues, metaoffsets)); } return rtr; }
for(ScoredDoc doc : results) if (doc.getScore() == Double.NEGATIVE_INFINITY) continue; JSONObject result = new JSONObject(); result.put("rank", rank); result.put("docid", doc.getDocid()); result.put("score", doc.getScore()); for(String meta : metakeys) result.put(meta, doc.getMetadata(meta));
public ScoredDocList getResults() { ResultSet rs = this.getResultSet(); if (rs == null) throw new IllegalStateException("No resultset found"); final String[] metaKeys = rs.getMetaKeys(); Results rtr = new Results(metaKeys); Map<String,Integer> metaoffsets = new HashMap<>(); final int metaCount = metaKeys.length; if (metaKeys != null) for(int j=0;j<metaCount;j++) { metaoffsets.put(metaKeys[j], j); } final int size = rs.getResultSize(); final int[] docids = rs.getDocids(); final double[] scores = rs.getScores(); final short[] occ = rs.getOccurrences(); final String[][] meta = rs.allMetaItems(); for(int i=0;i<size;i++) { String[] metaValues = new String[metaCount]; for(int j=0;j<metaCount;j++) metaValues[j] = meta[j][i]; rtr.add(new ScoredDoc(docids[i], scores[i], occ[i], metaValues, metaoffsets)); } return rtr; }