private static ParamInfo parseTerm(FunctionQParser fp) throws SyntaxError { ParamInfo paramInfo = new ParamInfo(); paramInfo.field = fp.parseArg(); String textVal = fp.parseArg(); if(textVal == null || textVal.trim().length() == 0){ return paramInfo; } if(fp.hasMoreArguments()){ String similarity = fp.parseArg().toLowerCase().trim(); if( !similarity.equals(SimilarityType.DOC_LEN) && !similarity.equals(SimilarityType.PARAM_LEN) && !similarity.equals(SimilarityType.DICE) && !similarity.equals(SimilarityType.JACCARD)){ log.error(String.format("Invalid similarity class: %s. Defaulting to %s", similarity, SimilarityType.DOC_LEN)); similarity = SimilarityType.DOC_LEN; } paramInfo.similarity = similarity; } // need to do analysis on the term Analyzer analyzer = fp.getReq().getSchema().getIndexAnalyzer(); paramInfo.analyzer = analyzer; try { List<String> terms = TermExtractionHelper.getTermsFromString(analyzer, paramInfo.field, textVal); paramInfo.terms = new HashSet<String>(terms); } catch (IOException e) { SolrException.log(log, "Exception during debug", e); return null; } return paramInfo; }
public ValueSource parse(FunctionQParser fp) throws SyntaxError { String indexedField = fp.parseArg(); Analyzer analyzer = fp.getReq().getSchema().getIndexAnalyzer(); return new FieldIndexedValueSource(indexedField, analyzer); } }
public ValueSource parse(FunctionQParser fp) throws SyntaxError { String indexedField = fp.parseArg(); Analyzer analyzer = fp.getReq().getSchema().getIndexAnalyzer(); return new FieldLenValueSource(indexedField, analyzer); } }
Analyzer analyzer = fp.getReq().getSchema().getIndexAnalyzer(); TokenStream ts = analyzer.tokenStream(fieldInfo.field, new StringReader(textVal)); try {
@Override public ValueSource parse(FunctionQParser fp) throws SyntaxError { FieldInfo fieldInfo = parseTerm(fp); if(fieldInfo == null){ return new DoubleConstValueSource(-1.0d); } if(fieldInfo.terms.size() == 0){ return new DoubleConstValueSource(0.0d); } ValueSource[] matches = new ValueSource[fieldInfo.terms.size()]; for(int i = 0; i < fieldInfo.terms.size(); i++){ TermInfo termInfo = fieldInfo.terms.get(i); matches[i] = new BinaryTermExistsValueSource(fieldInfo.field, termInfo.indexedBytes.utf8ToString(), fieldInfo.indexedField, termInfo.indexedBytes); } final SumFloatFunction sum = new SumFloatFunction(matches); ValueSource length = null; if(fieldInfo.useDocLength){ Analyzer analyzer = fp.getReq().getSchema().getIndexAnalyzer(); length = new FieldLenValueSource(fieldInfo.indexedField, analyzer); } else{ length = new DoubleConstValueSource(fieldInfo.terms.size()); } return new DivFloatFunction(sum, length); }
IndexReader reader = searcher.getIndexReader(); if(analyzer == null) { this.analyzer = searcher.getSchema().getIndexAnalyzer();
@Override public void process(ResponseBuilder rb) throws IOException { if(isEnabled(rb)){ long startTime = System.currentTimeMillis(); SolrParams params = rb.req.getParams(); int topN = getTopN(params); boolean binary = getBinary(params); boolean logTfs = getLogTfs(params); boolean includeExisting = getIncludeExisting(params); final SolrIndexSearcher searcher = rb.req.getSearcher(); IndexReader ir = searcher.getIndexReader(); Analyzer analyzer = searcher.getSchema().getIndexAnalyzer(); DocListAndSet docs = rb.getResults(); DocIterator iterator = docs.docList.iterator(); String uniqueKeyField = searcher.getSchema().getUniqueKeyField().getName(); NamedList<NamedList<Double>> topPredictions = new NamedList<NamedList<Double>>(); while(iterator.hasNext()) { int docNum = iterator.nextDoc(); Map<String, Map<String,Integer>> tf = getFieldTermFrequencyCounts(fields, ir, analyzer, docNum); NamedList<Double> predictions = predict(tf, topN, binary, logTfs, includeExisting); String uniqueFieldValue = getUniqueKeyFieldValue(ir, analyzer, uniqueKeyField, docNum); topPredictions.add(String.format("%s:%s", uniqueKeyField, uniqueFieldValue), predictions); } long duration = System.currentTimeMillis() - startTime; NamedList<Object> results = new NamedList<Object>(); results.add("Time", duration); results.add("values", topPredictions); rb.rsp.add(getPrefix(), results); } }
mlt.setAnalyzer( searcher.getSchema().getIndexAnalyzer() );
moreLikeThis.setPayloadFields(payloadFields); moreLikeThis.setAnalyzer(searcher.getSchema().getIndexAnalyzer());