while (it.hasNext()) docIds.add(it.next());
@Override public boolean hasNext() { if (iterator().hasNext()) { return true; } else { iterator = null; currentState = checkForConsumptionCompleteness; return currentState.hasNext(); } }
private Set<Integer> getDocidsFromQuery(Query query, SolrIndexSearcher searcher, int maxRows) throws SyntaxError, IOException { DocList match = searcher.getDocList(query, null, null, 0, maxRows, 0); // only get the first one... Set<Integer> docsIds = new HashSet<Integer>(); DocIterator iterator = match.iterator(); while (iterator.hasNext()) { docsIds.add(iterator.nextDoc()); } return docsIds; }
public void getResult(NamedList result, DocList docs, CollapseContext collapseContext) { for (DocIterator i = docs.iterator(); i.hasNext();) { int id = i.nextDoc(); Counter counter = documentHeadCount.get(id); if (counter == null) { continue; } NamedList collapsedGroupResult = getCollapseGroupResult(id, result); collapsedGroupResult.add("collapseCount", counter.getCount()); } } }
/** * Inefficient base implementation. * * @see BitDocSet#getBits */ public OpenBitSet getBits() { OpenBitSet bits = new OpenBitSet(); for (DocIterator iter = iterator(); iter.hasNext();) { bits.set(iter.nextDoc()); } return bits; };
private Map<String, Set<String>> findParentIdsForNodes(SolrIndexSearcher searcher, Collection<String> nodeIds) throws IOException { Map<String, Set<String>> parentIds = new HashMap<>(); LOGGER.debug("Looking up parents for {} nodes", nodeIds.size()); Query filter = buildFilterQuery(getNodeField(), nodeIds); LOGGER.trace("Filter query: {}", filter); DocSet docs = searcher.getDocSet(filter); for (DocIterator it = docs.iterator(); it.hasNext(); ) { Document doc = searcher.doc(it.nextDoc(), docFields); String nodeId = doc.get(getNodeField()); Set<String> parentIdValues = new HashSet<>(Arrays.asList(doc.getValues(parentField))); parentIds.put(nodeId, parentIdValues); // Record the label, if required if (isLabelRequired(nodeId)) { recordLabel(nodeId, doc.getValues(getLabelField())); } } return parentIds; }
@Override public void write(Writer w, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { SolrIndexSearcher searcher = req.getSearcher(); NamedList nl = rsp.getValues(); int sz = nl.size(); for (int li = 0; li < sz; li++) { Object val = nl.getVal(li); if (val instanceof DocList) { //<co id="co.fuzzy.type-ahead.doclist"/> DocList dl = (DocList) val; DocIterator iterator = dl.iterator(); w.append("<ul>\n"); while (iterator.hasNext()) { int id = iterator.nextDoc(); Document doc = searcher.doc(id, fields); //<co id="co.fuzzy.type-ahead.search"/> String name = doc.get("word"); w.append("<li>" + name + "</li>\n"); } w.append("</ul>\n"); } } } }
public void getResult(NamedList result, DocList docs, CollapseContext collapseContext) { Map<Integer, CollapseGroup> docHeadCollapseGroupAssoc = collapseContext.getDocumentHeadCollapseGroupAssociation(); for (DocIterator i = docs.iterator(); i.hasNext();) { int id = i.nextDoc(); CollapseGroup collapseGroup = docHeadCollapseGroupAssoc.get(id); if (collapseGroup == null) { continue; } NamedList collapsedGroupResult = getCollapseGroupResult(id, result); String fieldValue = collapseFieldType.indexedToReadable(collapseGroup.getKey()); collapsedGroupResult.add("fieldValue", fieldValue); } }
public boolean equals(Object obj) { if (!(obj instanceof DocSet)) return false; DocSet other = (DocSet)obj; if (this.size() != other.size()) return false; if (this instanceof DocList && other instanceof DocList) { // compare ordering DocIterator i1=this.iterator(); DocIterator i2=other.iterator(); while(i1.hasNext() && i2.hasNext()) { if (i1.nextDoc() != i2.nextDoc()) return false; } return true; // don't compare matches } // if (this.size() != other.size()) return false; return this.getBits().equals(other.getBits()); }
@Override public DocSet andNot(DocSet other) { OpenBitSet newbits = (OpenBitSet)(bits.clone()); if (other instanceof BitDocSet) { newbits.andNot(((BitDocSet)other).bits); } else { DocIterator iter = other.iterator(); while (iter.hasNext()) newbits.clear(iter.nextDoc()); } return new BitDocSet(newbits); }
@Override public DocSet union(DocSet other) { OpenBitSet newbits = (OpenBitSet)(bits.clone()); if (other instanceof BitDocSet) { newbits.union(((BitDocSet)other).bits); } else { DocIterator iter = other.iterator(); while (iter.hasNext()) newbits.set(iter.nextDoc()); } return new BitDocSet(newbits); }
public void getResult(NamedList result, DocList docs, CollapseContext collapseContext) { Map<Integer, CollapseGroup> docHeadCollapseGroupAssoc = collapseContext.getDocumentHeadCollapseGroupAssociation(); for (DocIterator i = docs.iterator(); i.hasNext();) { int id = i.nextDoc(); CollapseGroup collapseGroup = docHeadCollapseGroupAssoc.get(id); if (collapseGroup == null) { continue; } for (AggregateField aggregateField : functions.keySet()) { AggregateFunction function = functions.get(aggregateField); String functionResult = function.calculate(collapseGroup); if (functionResult != null) { NamedList collapsedGroupResult = getCollapseGroupResult(id, result); NamedList aggragateFunctions = (NamedList) collapsedGroupResult.get("aggregate"); if (aggragateFunctions == null) { aggragateFunctions = new NamedList<Object>(); collapsedGroupResult.add("aggregate", aggragateFunctions); } aggragateFunctions.add(aggregateField.getUniqueName(), functionResult); } } } }
/** * Returns a random set of documents from the index. Mainly for testing purposes. * * @param req * @param rsp * @throws IOException */ private void handleRandomSearch(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { SolrIndexSearcher searcher = req.getSearcher(); Query query = new MatchAllDocsQuery(); DocList docList = searcher.getDocList(query, getFilterQueries(req), Sort.RELEVANCE, 0, numberOfCandidateResults, 0); int paramRows = Math.min(req.getParams().getInt("rows", defaultNumberOfResults), docList.size()); if (docList.size() < 1) { rsp.add("Error", "No documents in index"); } else { LinkedList list = new LinkedList(); while (list.size() < paramRows) { DocList auxList = docList.subset((int) (Math.random() * docList.size()), 1); Document doc = null; for (DocIterator it = auxList.iterator(); it.hasNext(); ) { doc = searcher.doc(it.nextDoc()); } if (!list.contains(doc)) { list.add(doc); } } rsp.addResponse(list); } }
private MLTResult expandQueryAndReExecute(SolrQueryResponse rsp, SolrParams params, int maxDocumentsToMatch, int flags, String q, Query seedQuery, SortSpec sortSpec, List<Query> targetFqFilters, List<Query> mltFqFilters, SolrIndexSearcher searcher, UnsupervisedFeedbackHelper uff, int start, int rows) throws IOException, SyntaxError { boolean includeMatch = params.getBool(UnsupervisedFeedbackParams.MATCH_INCLUDE, true); int matchOffset = params.getInt(UnsupervisedFeedbackParams.MATCH_OFFSET, 0); // Find the base match DocList match = searcher.getDocList(seedQuery, targetFqFilters, null, matchOffset, maxDocumentsToMatch, flags); // only get the first one... if(match.matches() == 0){ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.format("Unsupervised feedback handler was unable to find any documents matching the seed query: '%s'.", q)); } if (includeMatch) { rsp.add("match", match); } // This is an iterator, but we only handle the first match DocIterator iterator = match.iterator(); MLTResult mltResult = null; if (iterator.hasNext()) { // do a MoreLikeThis query for each document in results mltResult = uff.expandQueryAndReExecute(iterator, seedQuery, start, rows, mltFqFilters, flags, sortSpec.getSort()); } return mltResult; }
private MLTResult getMoreLikeTheseFromQuery(SolrQueryResponse rsp, SolrParams params, int flags, String q, Query query, SortSpec sortSpec, List<Query> targetFqFilters, List<Query> mltFqFilters, SolrIndexSearcher searcher, MoreLikeThisHelper mlt, int start, int rows) throws IOException, SyntaxError { boolean includeMatch = params.getBool(MoreLikeThisParams.MATCH_INCLUDE, true); int matchOffset = params.getInt(MoreLikeThisParams.MATCH_OFFSET, 0); // Find the base match DocList match = searcher.getDocList(query, targetFqFilters, null, matchOffset, 10000, flags); // only get the first one... if(match.matches() == 0){ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.format("MoreLikeThis was unable to find any documents matching the query: '%s'.", q)); } if (includeMatch) { rsp.add("match", match); } // This is an iterator, but we only handle the first match DocIterator iterator = match.iterator(); if (iterator.hasNext()) { // do a MoreLikeThis query for each document in results return mlt.getMoreLikeTheseFromDocs(iterator, start, rows, mltFqFilters, flags, sortSpec.getSort()); } return null; }
public NamedList<DocList> getMoreLikeThese( DocList docs, int rows, int flags ) throws IOException { IndexSchema schema = searcher.getSchema(); NamedList<DocList> mlt = new SimpleOrderedMap<DocList>(); DocIterator iterator = docs.iterator(); while( iterator.hasNext() ) { int id = iterator.nextDoc(); DocListAndSet sim = getMoreLikeThis( id, 0, rows, null, null, flags ); String name = schema.printableUniqueKey( reader.document( id ) ); mlt.add(name, sim.docList); } return mlt; }
List<Integer> ids = new ArrayList<Integer>(); while(iterator.hasNext()) { int id = iterator.nextDoc(); Document doc = reader.document(id);
public MLTResult expandQueryAndReExecute(DocIterator iterator, Query seedQuery, int start, int rows, List<Query> filters, int flags, Sort lsort) throws IOException, SyntaxError while(iterator.hasNext()) { ids.add(iterator.nextDoc());
protected DocList sortDocSet(DocSet set, Sort sort, int nDocs) throws IOException { // bit of a hack to tell if a set is sorted - do it better in the futute. boolean inOrder = set instanceof BitDocSet || set instanceof SortedIntDocSet; TopDocsCollector topCollector = TopFieldCollector.create(sort, nDocs, false, false, false, inOrder); DocIterator iter = set.iterator(); int base=0; int end=0; int readerIndex = -1; SolrIndexReader r=null; while(iter.hasNext()) { int doc = iter.nextDoc(); while (doc>=end) { r = reader.getLeafReaders()[++readerIndex]; base = reader.getLeafOffsets()[readerIndex]; end = base + r.maxDoc(); topCollector.setNextReader(r, base); // we should never need to set the scorer given the settings for the collector } topCollector.collect(doc-base); } TopDocs topDocs = topCollector.topDocs(0, nDocs); int nDocsReturned = topDocs.scoreDocs.length; int[] ids = new int[nDocsReturned]; for (int i=0; i<nDocsReturned; i++) { ScoreDoc scoreDoc = topDocs.scoreDocs[i]; ids[i] = scoreDoc.doc; } return new DocSlice(0,nDocsReturned,ids,null,topDocs.totalHits,0.0f); }
@Override public void process(ResponseBuilder rb) throws IOException { if(isEnabled(rb)){ long startTime = System.currentTimeMillis(); SolrParams params = rb.req.getParams(); int topN = getTopN(params); boolean binary = getBinary(params); boolean logTfs = getLogTfs(params); boolean includeExisting = getIncludeExisting(params); final SolrIndexSearcher searcher = rb.req.getSearcher(); IndexReader ir = searcher.getIndexReader(); Analyzer analyzer = searcher.getSchema().getIndexAnalyzer(); DocListAndSet docs = rb.getResults(); DocIterator iterator = docs.docList.iterator(); String uniqueKeyField = searcher.getSchema().getUniqueKeyField().getName(); NamedList<NamedList<Double>> topPredictions = new NamedList<NamedList<Double>>(); while(iterator.hasNext()) { int docNum = iterator.nextDoc(); Map<String, Map<String,Integer>> tf = getFieldTermFrequencyCounts(fields, ir, analyzer, docNum); NamedList<Double> predictions = predict(tf, topN, binary, logTfs, includeExisting); String uniqueFieldValue = getUniqueKeyFieldValue(ir, analyzer, uniqueKeyField, docNum); topPredictions.add(String.format("%s:%s", uniqueKeyField, uniqueFieldValue), predictions); } long duration = System.currentTimeMillis() - startTime; NamedList<Object> results = new NamedList<Object>(); results.add("Time", duration); results.add("values", topPredictions); rb.rsp.add(getPrefix(), results); } }