for (int j = origStart + 1; j < end; j++) { double[] probs = model.eval(cg.getContext(tok, j - origStart)); String best = model.getBestOutcome(probs); tokenProb *= probs[model.getIndex(best)]; if (best.equals(TokenizerME.SPLIT)) { newTokens.add(new Span(start, j));
@Override public Language[] predictLanguages(CharSequence content) { double[] eval = model.getMaxentModel().eval(mContextGenerator.getContext(content.toString())); Language[] arr = new Language[eval.length]; for (int i = 0; i < eval.length; i++) { arr[i] = new Language(model.getMaxentModel().getOutcome(i), eval[i]); } Arrays.sort(arr, (o1, o2) -> Double.compare(o2.getConfidence(), o1.getConfidence())); return arr; }
@Override public String[] getOutcomes() { String[] outcomes = new String[model.getNumOutcomes()]; for (int i = 0; i < model.getNumOutcomes(); i++) { outcomes[i] = model.getOutcome(i); } return outcomes; } }
private Parser(MaxentModel buildModel, MaxentModel attachModel, MaxentModel checkModel, POSTagger tagger, Chunker chunker, HeadRules headRules, int beamSize, double advancePercentage) { super(tagger,chunker,headRules,beamSize,advancePercentage); this.buildModel = buildModel; this.attachModel = attachModel; this.checkModel = checkModel; this.buildContextGenerator = new BuildContextGenerator(); this.attachContextGenerator = new AttachContextGenerator(punctSet); this.checkContextGenerator = new CheckContextGenerator(punctSet); this.bprobs = new double[buildModel.getNumOutcomes()]; this.aprobs = new double[attachModel.getNumOutcomes()]; this.cprobs = new double[checkModel.getNumOutcomes()]; this.doneIndex = buildModel.getIndex(DONE); this.sisterAttachIndex = attachModel.getIndex(ATTACH_SISTER); this.daughterAttachIndex = attachModel.getIndex(ATTACH_DAUGHTER); this.nonAttachIndex = attachModel.getIndex(NON_ATTACH); attachments = new int[] {daughterAttachIndex,sisterAttachIndex}; this.completeIndex = checkModel.getIndex(Parser.COMPLETE); }
this.buildModel = buildModel; this.checkModel = checkModel; bprobs = new double[buildModel.getNumOutcomes()]; cprobs = new double[checkModel.getNumOutcomes()]; this.buildContextGenerator = new BuildContextGenerator(); this.checkContextGenerator = new CheckContextGenerator(); startTypeMap = new HashMap<>(); contTypeMap = new HashMap<>(); for (int boi = 0, bon = buildModel.getNumOutcomes(); boi < bon; boi++) { String outcome = buildModel.getOutcome(boi); if (outcome.startsWith(START)) { topStartIndex = buildModel.getIndex(TOP_START); completeIndex = checkModel.getIndex(COMPLETE); incompleteIndex = checkModel.getIndex(INCOMPLETE);
List<Parse> newParsesList = new ArrayList<>(buildModel.getNumOutcomes()); buildModel.eval(buildContextGenerator.getContext(children, advanceNodeIndex), bprobs); double bprobSum = 0; while (bprobSum < probMass) { bprobs[max] = 0; //zero out so new max can be found bprobSum += bprob; String tag = buildModel.getOutcome(max); checkModel.eval(checkContextGenerator.getContext( collapsePunctuation(newParse1.getChildren(),punctSet), lastStartType, lastStartIndex, advanceNodeIndex), cprobs);
/** * Categorize the given text provided as tokens along with * the provided extra information * * @param text text tokens to categorize * @param extraInformation additional information */ @Override public double[] categorize(String[] text, Map<String, Object> extraInformation) { return model.getMaxentModel().eval( mContextGenerator.getContext(text, extraInformation)); }
private void testModel(MaxentModel model, Event event, double higher_probability) { double[] outcomes = model.eval(event.getContext()); String outcome = model.getBestOutcome(outcomes); Assert.assertEquals(2, outcomes.length); Assert.assertEquals(event.getOutcome(), outcome); if (event.getOutcome().equals(model.getOutcome(0))) { Assert.assertEquals(higher_probability, outcomes[0], 0.0001); } if (!event.getOutcome().equals(model.getOutcome(0))) { Assert.assertEquals(1.0 - higher_probability, outcomes[0], 0.0001); } if (event.getOutcome().equals(model.getOutcome(1))) { Assert.assertEquals(higher_probability, outcomes[1], 0.0001); } if (!event.getOutcome().equals(model.getOutcome(1))) { Assert.assertEquals(1.0 - higher_probability, outcomes[1], 0.0001); } }
String[] contexts = smallTest.split(" "); float[] values = RealValueFileEventStream.parseContexts(contexts); double[] smallResults = smallModel.eval(contexts, values); String smallResultString = smallModel.getAllOutcomes(smallResults); System.out.println("smallResults: " + smallResultString); contexts = largeTest.split(" "); values = RealValueFileEventStream.parseContexts(contexts); double[] largeResults = largeModel.eval(contexts, values); String largeResultString = largeModel.getAllOutcomes(largeResults); System.out.println("largeResults: " + largeResultString); for (int i = 0; i < smallResults.length; i++) { System.out.println(String.format( "classifiy with smallModel: %1$s = %2$f", smallModel.getOutcome(i), smallResults[i])); System.out.println(String.format( "classifiy with largeModel: %1$s = %2$f", largeModel.getOutcome(i), largeResults[i])); Assert.assertEquals(smallResults[i], largeResults[i], 0.01f);
public int getIndex(String category) { return model.getMaxentModel().getIndex(category); }
public String getBestCategory(double[] outcome) { return model.getMaxentModel().getBestOutcome(outcome); }
this.model.getContextEntries()); double[] outcomeProbs = model.getWSDMaxentModel().eval(context); outcome = model.getWSDMaxentModel().getBestOutcome(outcomeProbs); this.model.getContextEntries()); double[] outcomeProbs = model.getWSDMaxentModel().eval(context); outcome = model.getWSDMaxentModel().getBestOutcome(outcomeProbs);
public BeamSearch(int size, MaxentModel model, int cacheSize) { this.size = size; this.model = model; if (cacheSize > 0) { contextsCache = new Cache<>(cacheSize); } this.probs = new double[model.getNumOutcomes()]; }
public String getCategory(int index) { return model.getMaxentModel().getOutcome(index); }
public String getAllResults(double[] results) { return model.getMaxentModel().getAllOutcomes(results); }
List<Parse> newParsesList = new ArrayList<>(buildModel.getNumOutcomes()); buildModel.eval(buildContextGenerator.getContext(children, advanceNodeIndex), bprobs); double bprobSum = 0; while (bprobSum < probMass) { bprobs[max] = 0; //zero out so new max can be found bprobSum += bprob; String tag = buildModel.getOutcome(max); checkModel.eval(checkContextGenerator.getContext( collapsePunctuation(newParse1.getChildren(),punctSet), lastStartType, lastStartIndex, advanceNodeIndex), cprobs);
@Override protected void advanceTop(Parse p) { buildModel.eval(buildContextGenerator.getContext(p.getChildren(), 0), bprobs); p.addProb(Math.log(bprobs[topStartIndex])); checkModel.eval(checkContextGenerator.getContext(p.getChildren(), TOP_NODE, 0, 0), cprobs); p.addProb(Math.log(cprobs[completeIndex])); p.setType(TOP_NODE); }
this.buildModel = buildModel; this.checkModel = checkModel; bprobs = new double[buildModel.getNumOutcomes()]; cprobs = new double[checkModel.getNumOutcomes()]; this.buildContextGenerator = new BuildContextGenerator(); this.checkContextGenerator = new CheckContextGenerator(); startTypeMap = new HashMap<>(); contTypeMap = new HashMap<>(); for (int boi = 0, bon = buildModel.getNumOutcomes(); boi < bon; boi++) { String outcome = buildModel.getOutcome(boi); if (outcome.startsWith(START)) { topStartIndex = buildModel.getIndex(TOP_START); completeIndex = checkModel.getIndex(COMPLETE); incompleteIndex = checkModel.getIndex(INCOMPLETE);
private Parser(MaxentModel buildModel, MaxentModel attachModel, MaxentModel checkModel, POSTagger tagger, Chunker chunker, HeadRules headRules, int beamSize, double advancePercentage) { super(tagger,chunker,headRules,beamSize,advancePercentage); this.buildModel = buildModel; this.attachModel = attachModel; this.checkModel = checkModel; this.buildContextGenerator = new BuildContextGenerator(); this.attachContextGenerator = new AttachContextGenerator(punctSet); this.checkContextGenerator = new CheckContextGenerator(punctSet); this.bprobs = new double[buildModel.getNumOutcomes()]; this.aprobs = new double[attachModel.getNumOutcomes()]; this.cprobs = new double[checkModel.getNumOutcomes()]; this.doneIndex = buildModel.getIndex(DONE); this.sisterAttachIndex = attachModel.getIndex(ATTACH_SISTER); this.daughterAttachIndex = attachModel.getIndex(ATTACH_DAUGHTER); this.nonAttachIndex = attachModel.getIndex(NON_ATTACH); attachments = new int[] {daughterAttachIndex,sisterAttachIndex}; this.completeIndex = checkModel.getIndex(Parser.COMPLETE); }
public int getIndex(String category) { return model.getMaxentModel().getIndex(category); }