@Override public Event read() throws IOException { String line; if ((line = reader.readLine()) != null) { int si = line.indexOf(' '); String outcome = line.substring(0, si); String[] contexts = line.substring(si + 1).split(" "); float[] values = parseContexts(contexts); return new Event(outcome, contexts, values); } return null; } }
@Test public void testLastLineBug() throws IOException { try (RealValueFileEventStream rvfes = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/io/rvfes-bug-data-ok.txt")) { indexer.index(rvfes); } Assert.assertEquals(1, indexer.getOutcomeLabels().length); try (RealValueFileEventStream rvfes = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/io/rvfes-bug-data-broken.txt")) { indexer.index(rvfes); } Assert.assertEquals(1, indexer.getOutcomeLabels().length); } }
private Event createEvent(String obs) { int lastSpace = obs.lastIndexOf(' '); if (lastSpace == -1) return null; else { String[] contexts = obs.substring(0,lastSpace).split("\\s+"); float[] values = RealValueFileEventStream.parseContexts(contexts); return new Event(obs.substring(lastSpace + 1),contexts,values); } }
@Test public void testTrainModelReturnsAQNModel() throws Exception { // given RealValueFileEventStream rvfes1 = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt"); testDataIndexer.index(rvfes1); // when QNModel trainedModel = new QNTrainer(false).trainModel(ITERATIONS, testDataIndexer); // then Assert.assertNotNull(trainedModel); }
private Event createEvent(String obs) { int lastSpace = obs.lastIndexOf(' '); if (lastSpace == -1) return null; else { String[] contexts = obs.substring(0,lastSpace).split("\\s+"); float[] values = RealValueFileEventStream.parseContexts(contexts); return new Event(obs.substring(lastSpace + 1),contexts,values); } }
@Test public void testValueAtNonInitialPoint01() throws IOException { // given RealValueFileEventStream rvfes1 = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt", "UTF-8"); testDataIndexer.index(rvfes1); NegLogLikelihood objectFunction = new NegLogLikelihood(testDataIndexer); // when double[] nonInitialPoint = new double[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }; double value = objectFunction.valueAt(nonInitialPoint); double expectedValue = 13.862943611198894; // then Assert.assertEquals(expectedValue, value, TOLERANCE01); }
private Event createEvent(String obs) { int lastSpace = obs.lastIndexOf(' '); if (lastSpace == -1) return null; else { String[] contexts = obs.substring(0,lastSpace).split("\\s+"); float[] values = RealValueFileEventStream.parseContexts(contexts); return new Event(obs.substring(lastSpace + 1),contexts,values); } }
@Test public void testInitialSanity() throws IOException { // given RealValueFileEventStream rvfes1 = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt", "UTF-8"); testDataIndexer.index(rvfes1); NegLogLikelihood objectFunction = new NegLogLikelihood(testDataIndexer); // when double[] initial = objectFunction.getInitialPoint(); // then for (double anInitial : initial) { Assert.assertEquals(0.0, anInitial, TOLERANCE01); } }
@Override public Event read() throws IOException { String line; if ((line = reader.readLine()) != null) { int si = line.indexOf(' '); String outcome = line.substring(0, si); String[] contexts = line.substring(si + 1).split(" "); float[] values = parseContexts(contexts); return new Event(outcome, contexts, values); } return null; } }
@Test public void testModel() throws IOException { // given RealValueFileEventStream rvfes1 = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt"); testDataIndexer.index(rvfes1); // when QNModel trainedModel = new QNTrainer(15, true).trainModel( ITERATIONS, testDataIndexer); Assert.assertFalse(trainedModel.equals(null)); }
float[] values = RealValueFileEventStream.parseContexts(contexts); double[] smallResults = smallModel.eval(contexts, values); values = RealValueFileEventStream.parseContexts(contexts); double[] largeResults = largeModel.eval(contexts, values);
@Test public void testGradientSanity() throws IOException { // given RealValueFileEventStream rvfes1 = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt", "UTF-8"); testDataIndexer.index(rvfes1); NegLogLikelihood objectFunction = new NegLogLikelihood(testDataIndexer); // when double[] initial = objectFunction.getInitialPoint(); double[] gradientAtInitial = objectFunction.gradientAt(initial); // then Assert.assertNotNull(gradientAtInitial); }
@Override public Event read() throws IOException { String line; if ((line = reader.readLine()) != null) { int si = line.indexOf(' '); String outcome = line.substring(0, si); String[] contexts = line.substring(si + 1).split(" "); float[] values = parseContexts(contexts); return new Event(outcome, contexts, values); } return null; } }
@Test public void testValueAtInitialPoint() throws IOException { // given RealValueFileEventStream rvfes1 = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt", "UTF-8"); testDataIndexer.index(rvfes1); NegLogLikelihood objectFunction = new NegLogLikelihood(testDataIndexer); // when double value = objectFunction.valueAt(objectFunction.getInitialPoint()); double expectedValue = 13.86294361; // then Assert.assertEquals(expectedValue, value, TOLERANCE01); }
@Test public void testInTinyDevSet() throws Exception { // given RealValueFileEventStream rvfes1 = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt"); testDataIndexer.index(rvfes1);; // when QNModel trainedModel = new QNTrainer(15, true).trainModel(ITERATIONS, testDataIndexer); String[] features2Classify = new String[] { "feature2","feature3", "feature3", "feature3","feature3", "feature3", "feature3","feature3", "feature3", "feature3","feature3", "feature3"}; double[] eval = trainedModel.eval(features2Classify); // then Assert.assertNotNull(eval); }
@Test public void testSerdeModel() throws IOException { RealValueFileEventStream rvfes1 = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt"); testDataIndexer.index(rvfes1);
GISModel realModel; GISTrainer gisTrainer = new GISTrainer(); try (RealValueFileEventStream rvfes1 = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt")) { testDataIndexer.index(rvfes1);
@Test public void testGradientAtInitialPoint() throws IOException { // given RealValueFileEventStream rvfes1 = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt", "UTF-8"); testDataIndexer.index(rvfes1); NegLogLikelihood objectFunction = new NegLogLikelihood(testDataIndexer); // when double[] gradientAtInitialPoint = objectFunction.gradientAt(objectFunction.getInitialPoint()); double[] expectedGradient = new double[] { -9.0, -14.0, -17.0, 20.0, 8.5, 9.0, 14.0, 17.0, -20.0, -8.5 }; // then Assert.assertTrue(compareDoubleArray(expectedGradient, gradientAtInitialPoint, testDataIndexer, TOLERANCE01)); }
@Test public void testDomainDimensionSanity() throws IOException { // given RealValueFileEventStream rvfes1 = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt", "UTF-8"); testDataIndexer.index(rvfes1); NegLogLikelihood objectFunction = new NegLogLikelihood(testDataIndexer); // when int correctDomainDimension = testDataIndexer.getPredLabels().length * testDataIndexer.getOutcomeLabels().length; // then Assert.assertEquals(correctDomainDimension, objectFunction.getDimension()); }
@Test public void testValueAtNonInitialPoint02() throws IOException { // given RealValueFileEventStream rvfes1 = new RealValueFileEventStream( "src/test/resources/data/opennlp/maxent/real-valued-weights-training-data.txt", "UTF-8"); testDataIndexer.index(rvfes1); NegLogLikelihood objectFunction = new NegLogLikelihood(testDataIndexer); // when double[] nonInitialPoint = new double[] { 3, 2, 3, 2, 3, 2, 3, 2, 3, 2 }; double value = objectFunction.valueAt(dealignDoubleArrayForTestData(nonInitialPoint, testDataIndexer.getPredLabels(), testDataIndexer.getOutcomeLabels())); double expectedValue = 53.163219721099026; // then Assert.assertEquals(expectedValue, value, TOLERANCE02); }