public void testForwardBackward () { SumLatticeDefault lattice = new SumLatticeDefault (transducer, seq); System.out.println ("weight= "+lattice.getTotalWeight()); assertTrue (lattice.getTotalWeight() == seqWeight); }
private static void outputLatticeRows (PrintWriter out, SumLatticeDefault lattice, int start, int end) Transducer ducer = lattice.getTransducer (); for (int k = 0; k < ducer.numStates(); k++) { Transducer.State state = ducer.getState (k); out.println (" <td class=\"label\">α("+state.getName()+")</td>"); for (int ip = start; ip < end; ip++) { out.print ("<td>"+f.format (lattice.getAlpha (ip+1, state))+"</td>"); out.println (" <td class=\"label\">β("+state.getName()+")</td>"); for (int ip = start; ip < end; ip++) { out.print ("<td>"+f.format (lattice.getBeta (ip+1, state))+"</td>"); out.println (" <td class=\"label\">γ("+state.getName()+")</td>"); for (int ip = start; ip < end; ip++) { out.print ("<td>"+f.format (lattice.getGammaWeight(ip+1, state))+"</td>");
public double getBeta (int ip, State s) { LatticeNode node = getLatticeNode (ip, s.getIndex ()); return node.beta; }
public SumLattice newSumLattice (Transducer trans, Sequence input, Sequence output, Transducer.Incrementor incrementor, boolean saveXis, LabelAlphabet outputAlphabet) { return new SumLatticeDefault (trans, input, output, incrementor, saveXis, outputAlphabet); }
/** Calculates the confidence in the tagging of a {@link Segment}. @return 0-1 confidence value. higher = more confident. */ public double estimateConfidenceFor (Segment segment, SumLatticeDefault cachedLattice) { Sequence predSequence = segment.getPredicted (); Sequence input = segment.getInput (); SumLatticeDefault lattice = (cachedLattice==null) ? new SumLatticeDefault (model, input) : cachedLattice; double confidence = 1; for (int i=segment.getStart(); i <= segment.getEnd(); i++) confidence *= lattice.getGammaProbability (i+1, model.getState (stateIndexOfString ((String)predSequence.get (i)))); return confidence; }
public void testXis() { Pipe p = makeSpacePredictionPipe(); InstanceList instances = new InstanceList(p); instances.addThruPipe(new ArrayIterator(data)); CRF crf1 = new CRF(p, null); crf1.addFullyConnectedStatesForLabels(); CRFTrainerByLabelLikelihood crft1 = new CRFTrainerByLabelLikelihood( crf1); crft1.train(instances, 10); // Let's get some parameters Instance inst = instances.get(0); Sequence input = (Sequence) inst.getData(); SumLatticeDefault lattice = new SumLatticeDefault(crf1, input, (Sequence) inst.getTarget(), null, true); for (int ip = 0; ip < lattice.length() - 1; ip++) { for (int i = 0; i < crf1.numStates(); i++) { Transducer.State state = crf1.getState(i); Transducer.TransitionIterator it = state.transitionIterator( input, ip); double gamma = lattice.getGammaProbability(ip, state); double xiSum = 0; while (it.hasNext()) { Transducer.State dest = it.nextState(); double xi = lattice.getXiProbability(ip, state, dest); xiSum += xi; } assertEquals(gamma, xiSum, 1e-5); } } }
public SumLattice newSumLattice (Transducer trans, Sequence input, Sequence output, Transducer.Incrementor incrementor, boolean saveXis, LabelAlphabet outputAlphabet) { return new SumLatticeDefault (trans, input, output, incrementor, saveXis, outputAlphabet); }
/** Calculates the confidence in the tagging of a {@link Segment}. @return 0-1 confidence value. higher = more confident. */ public double estimateConfidenceFor (Segment segment, SumLatticeDefault cachedLattice) { Sequence predSequence = segment.getPredicted (); Sequence input = segment.getInput (); SumLatticeDefault lattice = (cachedLattice==null) ? new SumLatticeDefault (model, input) : cachedLattice; double confidence = 1; for (int i=segment.getStart(); i <= segment.getEnd(); i++) confidence *= lattice.getGammaProbability (i+1, model.getState (stateIndexOfString ((String)predSequence.get (i)))); return confidence; }
public void testXis() { Pipe p = makeSpacePredictionPipe(); InstanceList instances = new InstanceList(p); instances.addThruPipe(new ArrayIterator(data)); CRF crf1 = new CRF(p, null); crf1.addFullyConnectedStatesForLabels(); CRFTrainerByLabelLikelihood crft1 = new CRFTrainerByLabelLikelihood( crf1); crft1.train(instances, 10); // Let's get some parameters Instance inst = instances.get(0); Sequence input = (Sequence) inst.getData(); SumLatticeDefault lattice = new SumLatticeDefault(crf1, input, (Sequence) inst.getTarget(), null, true); for (int ip = 0; ip < lattice.length() - 1; ip++) { for (int i = 0; i < crf1.numStates(); i++) { Transducer.State state = crf1.getState(i); Transducer.TransitionIterator it = state.transitionIterator( input, ip); double gamma = lattice.getGammaProbability(ip, state); double xiSum = 0; while (it.hasNext()) { Transducer.State dest = it.nextState(); double xi = lattice.getXiProbability(ip, state, dest); xiSum += xi; } assertEquals(gamma, xiSum, 1e-5); } } }
public void testForwardBackward () { SumLatticeDefault lattice = new SumLatticeDefault (transducer, seq); System.out.println ("weight= "+lattice.getTotalWeight()); assertTrue (lattice.getTotalWeight() == seqWeight); }
public SumLattice newSumLattice (Transducer trans, Sequence input, Sequence output, Transducer.Incrementor incrementor, boolean saveXis, LabelAlphabet outputAlphabet) { return new SumLatticeDefault (trans, input, output, incrementor, saveXis, outputAlphabet); }
private static void outputLatticeRows (PrintWriter out, SumLatticeDefault lattice, int start, int end) Transducer ducer = lattice.getTransducer (); for (int k = 0; k < ducer.numStates(); k++) { Transducer.State state = ducer.getState (k); out.println (" <td class=\"label\">α("+state.getName()+")</td>"); for (int ip = start; ip < end; ip++) { out.print ("<td>"+f.format (lattice.getAlpha (ip+1, state))+"</td>"); out.println (" <td class=\"label\">β("+state.getName()+")</td>"); for (int ip = start; ip < end; ip++) { out.print ("<td>"+f.format (lattice.getBeta (ip+1, state))+"</td>"); out.println (" <td class=\"label\">γ("+state.getName()+")</td>"); for (int ip = start; ip < end; ip++) { out.print ("<td>"+f.format (lattice.getGammaWeight(ip+1, state))+"</td>");
/** Calculates the confidence in the tagging of a {@link Segment}. @return 0-1 confidence value. higher = more confident. */ public double estimateConfidenceFor (Segment segment, SumLatticeDefault cachedLattice) { Sequence predSequence = segment.getPredicted (); Sequence input = segment.getInput (); SumLatticeDefault lattice = (cachedLattice==null) ? new SumLatticeDefault (model, input) : cachedLattice; double confidence = 1; for (int i=segment.getStart(); i <= segment.getEnd(); i++) confidence *= lattice.getGammaProbability (i+1, model.getState (stateIndexOfString ((String)predSequence.get (i)))); return confidence; }
public double getBeta (int ip, State s) { LatticeNode node = getLatticeNode (ip, s.getIndex ()); return node.beta; }
public void testEstimate () { transducer.setTrainable (true); SumLatticeDefault lattice = new SumLatticeDefault (transducer, seq); // used to have third argument: true double oldWeight = lattice.getTotalWeight (); transducer.estimate (); lattice = new SumLatticeDefault (transducer, seq); // used to have third argument: false double newWeight = lattice.getTotalWeight (); System.out.println ("oldWeight="+oldWeight+" newWeight="+newWeight); assertTrue (newWeight < oldWeight); }
public Void call() throws Exception { for (int ii = start; ii < end; ii++) { if (instancesWithConstraints.get(ii)) { Instance instance = data.get(ii); SumLatticeDefault lattice = new SumLatticeDefault( this.crf, (FeatureVectorSequence)instance.getData(), null, null, true); lattices.add(lattice); } else { lattices.add(null); } } return null; } }
private static void outputLatticeRows (PrintWriter out, SumLatticeDefault lattice, int start, int end) Transducer ducer = lattice.getTransducer (); for (int k = 0; k < ducer.numStates(); k++) { Transducer.State state = ducer.getState (k); out.println (" <td class=\"label\">α("+state.getName()+")</td>"); for (int ip = start; ip < end; ip++) { out.print ("<td>"+f.format (lattice.getAlpha (ip+1, state))+"</td>"); out.println (" <td class=\"label\">β("+state.getName()+")</td>"); for (int ip = start; ip < end; ip++) { out.print ("<td>"+f.format (lattice.getBeta (ip+1, state))+"</td>"); out.println (" <td class=\"label\">γ("+state.getName()+")</td>"); for (int ip = start; ip < end; ip++) { out.print ("<td>"+f.format (lattice.getGammaWeight(ip+1, state))+"</td>");
/** Calculates the confidence in the tagging of a {@link Segment}. @return 0-1 confidence value. higher = more confident. */ public double estimateConfidenceFor (Segment segment, SumLatticeDefault cachedLattice) { Sequence predSequence = segment.getPredicted (); Sequence input = segment.getInput (); SumLatticeDefault lattice = (cachedLattice==null) ? new SumLatticeDefault (model, input) : cachedLattice; double confidence = 0; for (int i=segment.getStart(); i <= segment.getEnd(); i++) { int stateIndex = stateIndexOfString((String)predSequence.get(i)); if (stateIndex == -1) // Unknown label. return 0.0; confidence += lattice.getGammaProbability(i+1, model.getState(stateIndex)); } return confidence/(double)segment.size(); }
public double getAlpha (int ip, State s) { LatticeNode node = getLatticeNode (ip, s.getIndex ()); return node.alpha; }
public void testEstimate () { transducer.setTrainable (true); SumLatticeDefault lattice = new SumLatticeDefault (transducer, seq); // used to have third argument: true double oldWeight = lattice.getTotalWeight (); transducer.estimate (); lattice = new SumLatticeDefault (transducer, seq); // used to have third argument: false double newWeight = lattice.getTotalWeight (); System.out.println ("oldWeight="+oldWeight+" newWeight="+newWeight); assertTrue (newWeight < oldWeight); }