public void testSpaceViewer () throws FileNotFoundException { Pipe pipe = TestMEMM.makeSpacePredictionPipe (); String[] data0 = { TestCRF.data[0] }; String[] data1 = { TestCRF.data[1] }; InstanceList training = new InstanceList (pipe); training.addThruPipe (new ArrayIterator (data0)); InstanceList testing = new InstanceList (pipe); testing.addThruPipe (new ArrayIterator (data1)); CRF crf = new CRF (pipe, null); crf.addFullyConnectedStatesForLabels (); CRFTrainerByLabelLikelihood crft = new CRFTrainerByLabelLikelihood (crf); crft.trainIncremental (training); CRFExtractor extor = hackCrfExtor (crf); Extraction extration = extor.extract (new ArrayIterator (data1)); PrintStream out = new PrintStream (new FileOutputStream (htmlFile)); LatticeViewer.extraction2html (extration, extor, out); out.close(); out = new PrintStream (new FileOutputStream (latticeFile)); LatticeViewer.extraction2html (extration, extor, out, true); out.close(); }
public void testSpaceViewer () throws FileNotFoundException { Pipe pipe = TestMEMM.makeSpacePredictionPipe (); String[] data0 = { TestCRF.data[0] }; String[] data1 = { TestCRF.data[1] }; InstanceList training = new InstanceList (pipe); training.addThruPipe (new ArrayIterator (data0)); InstanceList testing = new InstanceList (pipe); testing.addThruPipe (new ArrayIterator (data1)); CRF crf = new CRF (pipe, null); crf.addFullyConnectedStatesForLabels (); CRFTrainerByLabelLikelihood crft = new CRFTrainerByLabelLikelihood (crf); crft.trainIncremental (training); CRFExtractor extor = hackCrfExtor (crf); Extraction extration = extor.extract (new ArrayIterator (data1)); PrintStream out = new PrintStream (new FileOutputStream (htmlFile)); LatticeViewer.extraction2html (extration, extor, out); out.close(); out = new PrintStream (new FileOutputStream (latticeFile)); LatticeViewer.extraction2html (extration, extor, out, true); out.close(); }
boolean useSparseWeights) Pipe p = makeSpacePredictionPipe ();
boolean useSparseWeights) Pipe p = makeSpacePredictionPipe ();
public void doTestSpacePrediction(boolean testValueAndGradient) Pipe p = makeSpacePredictionPipe (); Pipe p2 = new TestMEMM2String();
public void doTestSpacePrediction(boolean testValueAndGradient) Pipe p = makeSpacePredictionPipe (); Pipe p2 = new TestMEMM2String();
public void testSpaceViewer () throws IOException { Pipe pipe = TestMEMM.makeSpacePredictionPipe (); String[] data0 = { TestCRF.data[0] }; String[] data1 = { TestCRF.data[1] }; InstanceList training = new InstanceList (pipe); training.addThruPipe (new ArrayIterator (data0)); InstanceList testing = new InstanceList (pipe); testing.addThruPipe (new ArrayIterator (data1)); CRF crf = new CRF (pipe, null); crf.addFullyConnectedStatesForLabels (); CRFTrainerByLabelLikelihood crft = new CRFTrainerByLabelLikelihood (crf); crft.trainIncremental (training); CRFExtractor extor = TestLatticeViewer.hackCrfExtor (crf); Extraction extraction = extor.extract (new ArrayIterator (data1)); if (!outputDir.exists ()) outputDir.mkdir (); DocumentViewer.writeExtraction (outputDir, extraction); }
public void testSpaceViewer () throws IOException { Pipe pipe = TestMEMM.makeSpacePredictionPipe (); String[] data0 = { TestCRF.data[0] }; String[] data1 = { TestCRF.data[1] }; InstanceList training = new InstanceList (pipe); training.addThruPipe (new ArrayIterator (data0)); InstanceList testing = new InstanceList (pipe); testing.addThruPipe (new ArrayIterator (data1)); CRF crf = new CRF (pipe, null); crf.addFullyConnectedStatesForLabels (); CRFTrainerByLabelLikelihood crft = new CRFTrainerByLabelLikelihood (crf); crft.trainIncremental (training); CRFExtractor extor = TestLatticeViewer.hackCrfExtor (crf); Extraction extraction = extor.extract (new ArrayIterator (data1)); if (!outputDir.exists ()) outputDir.mkdir (); DocumentViewer.writeExtraction (outputDir, extraction); }
public void testSpaceMaximizable () { Pipe p = makeSpacePredictionPipe (); InstanceList training = new InstanceList (p); // String[] data = { TestMEMM.data[0], }; // TestMEMM.data[1], TestMEMM.data[2], TestMEMM.data[3], }; // String[] data = { "ab" }; training.addThruPipe (new ArrayIterator (data)); // CRF4 memm = new CRF4 (p, null); MEMM memm = new MEMM (p, null); memm.addFullyConnectedStatesForLabels (); memm.addStartState(); memm.setWeightsDimensionAsIn(training); MEMMTrainer memmt = new MEMMTrainer (memm); // memm.gatherTrainingSets (training); // ANNOYING: Need to set up per-instance training sets memmt.train (training, 1); // Set weights dimension, gathers training sets, etc. // memm.print(); // memm.printGradient = true; // memm.printInstanceLists(); // memm.setGaussianPriorVariance (Double.POSITIVE_INFINITY); Optimizable.ByGradientValue mcrf = memmt.getOptimizableMEMM(training); TestOptimizable.setNumComponents (150); TestOptimizable.testValueAndGradient (mcrf); }
public void testSpaceMaximizable () { Pipe p = makeSpacePredictionPipe (); InstanceList training = new InstanceList (p); // String[] data = { TestMEMM.data[0], }; // TestMEMM.data[1], TestMEMM.data[2], TestMEMM.data[3], }; // String[] data = { "ab" }; training.addThruPipe (new ArrayIterator (data)); // CRF4 memm = new CRF4 (p, null); MEMM memm = new MEMM (p, null); memm.addFullyConnectedStatesForLabels (); memm.addStartState(); memm.setWeightsDimensionAsIn(training); MEMMTrainer memmt = new MEMMTrainer (memm); // memm.gatherTrainingSets (training); // ANNOYING: Need to set up per-instance training sets memmt.train (training, 1); // Set weights dimension, gathers training sets, etc. // memm.print(); // memm.printGradient = true; // memm.printInstanceLists(); // memm.setGaussianPriorVariance (Double.POSITIVE_INFINITY); Optimizable.ByGradientValue mcrf = memmt.getOptimizableMEMM(training); TestOptimizable.setNumComponents (150); TestOptimizable.testValueAndGradient (mcrf); }
public void testDualSpaceViewer () throws IOException Pipe pipe = TestMEMM.makeSpacePredictionPipe (); String[] data0 = { TestCRF.data[0] }; String[] data1 = TestCRF.data; Extraction e1 = extor.extract (new ArrayIterator (data1)); Pipe pipe2 = TestMEMM.makeSpacePredictionPipe (); InstanceList training2 = new InstanceList (pipe2); training2.addThruPipe (new ArrayIterator (data0));
public void testDualSpaceViewer () throws IOException Pipe pipe = TestMEMM.makeSpacePredictionPipe (); String[] data0 = { TestCRF.data[0] }; String[] data1 = TestCRF.data; Extraction e1 = extor.extract (new ArrayIterator (data1)); Pipe pipe2 = TestMEMM.makeSpacePredictionPipe (); InstanceList training2 = new InstanceList (pipe2); training2.addThruPipe (new ArrayIterator (data0));
public void testSpaceSerializable () throws IOException, ClassNotFoundException { Pipe p = makeSpacePredictionPipe (); InstanceList training = new InstanceList (p); training.addThruPipe (new ArrayIterator (data)); MEMM memm = new MEMM (p, null); memm.addFullyConnectedStatesForLabels (); memm.addStartState(); memm.setWeightsDimensionAsIn(training); MEMMTrainer memmt = new MEMMTrainer (memm); memmt.train (training, 10); MEMM memm2 = (MEMM) TestSerializable.cloneViaSerialization (memm); Optimizable.ByGradientValue mcrf1 = memmt.getOptimizableMEMM(training); double val1 = mcrf1.getValue (); Optimizable.ByGradientValue mcrf2 = memmt.getOptimizableMEMM(training); double val2 = mcrf2.getValue (); assertEquals (val1, val2, 1e-5); }
public void testSpaceSerializable () throws IOException, ClassNotFoundException { Pipe p = makeSpacePredictionPipe (); InstanceList training = new InstanceList (p); training.addThruPipe (new ArrayIterator (data)); MEMM memm = new MEMM (p, null); memm.addFullyConnectedStatesForLabels (); memm.addStartState(); memm.setWeightsDimensionAsIn(training); MEMMTrainer memmt = new MEMMTrainer (memm); memmt.train (training, 10); MEMM memm2 = (MEMM) TestSerializable.cloneViaSerialization (memm); Optimizable.ByGradientValue mcrf1 = memmt.getOptimizableMEMM(training); double val1 = mcrf1.getValue (); Optimizable.ByGradientValue mcrf2 = memmt.getOptimizableMEMM(training); double val2 = mcrf2.getValue (); assertEquals (val1, val2, 1e-5); }
public void disabledtestAddOrderNStates () Pipe p = makeSpacePredictionPipe ();
public void disabledtestAddOrderNStates () Pipe p = makeSpacePredictionPipe ();