public NegLogLikelihood(DataIndexer indexer) { // Get data from indexer. if (indexer instanceof OnePassRealValueDataIndexer) { this.values = indexer.getValues(); } else { this.values = null; } this.contexts = indexer.getContexts(); this.outcomeList = indexer.getOutcomeList(); this.numTimesEventsSeen = indexer.getNumTimesEventsSeen(); this.numOutcomes = indexer.getOutcomeLabels().length; this.numFeatures = indexer.getPredLabels().length; this.numContexts = this.contexts.length; this.dimension = numOutcomes * numFeatures; this.expectation = new double[numOutcomes]; this.tempSums = new double[numOutcomes]; this.gradient = new double[dimension]; }
/** * Evaluate the current model on training data set * @return model's training accuracy */ @Override public double evaluate(double[] parameters) { int[][] contexts = indexer.getContexts(); float[][] values = indexer.getValues(); int[] nEventsSeen = indexer.getNumTimesEventsSeen(); int[] outcomeList = indexer.getOutcomeList(); int nOutcomes = indexer.getOutcomeLabels().length; int nPredLabels = indexer.getPredLabels().length; int nCorrect = 0; int nTotalEvents = 0; for (int ei = 0; ei < contexts.length; ei++) { int[] context = contexts[ei]; float[] value = values == null ? null : values[ei]; double[] probs = new double[nOutcomes]; QNModel.eval(context, value, probs, nOutcomes, nPredLabels, parameters); int outcome = ArrayMath.argmax(probs); if (outcome == outcomeList[ei]) { nCorrect += nEventsSeen[ei]; } nTotalEvents += nEventsSeen[ei]; } return (double) nCorrect / nTotalEvents; } }
@Test public void testIndexWithNewline() throws IOException { String[] sentence = "He belongs to Apache \n Software Foundation .".split(" "); NameContextGenerator CG = new DefaultNameContextGenerator( (AdaptiveFeatureGenerator[]) null); NameSample nameSample = new NameSample(sentence, new Span[] { new Span(3, 7) }, false); ObjectStream<Event> eventStream = new NameFinderEventStream( ObjectStreamUtils.createObjectStream(nameSample), "org", CG, null); DataIndexer indexer = new TwoPassDataIndexer(); indexer.init(new TrainingParameters(Collections.emptyMap()), null); indexer.index(eventStream); Assert.assertEquals(5, indexer.getContexts().length); } }
public AbstractModel trainModel(DataIndexer di) { display("Incorporating indexed data for training... \n"); contexts = di.getContexts(); values = di.getValues(); numTimesEventsSeen = di.getNumTimesEventsSeen(); numEvents = di.getNumEvents(); numUniqueEvents = contexts.length; outcomeLabels = di.getOutcomeLabels(); outcomeList = di.getOutcomeList(); predLabels = di.getPredLabels(); numPreds = predLabels.length; numOutcomes = outcomeLabels.length; display("done.\n"); display("\tNumber of Event Tokens: " + numUniqueEvents + "\n"); display("\t Number of Outcomes: " + numOutcomes + "\n"); display("\t Number of Predicates: " + numPreds + "\n"); display("Computing model parameters...\n"); MutableContext[] finalParameters = findParameters(); display("...done.\n"); /* Create and return the model ****/ return new NaiveBayesModel(finalParameters, predLabels, outcomeLabels); }
public AbstractModel trainModel(int iterations, DataIndexer di, int cutoff, boolean useAverage) { display("Incorporating indexed data for training... \n"); contexts = di.getContexts(); values = di.getValues(); numTimesEventsSeen = di.getNumTimesEventsSeen(); numEvents = di.getNumEvents(); numUniqueEvents = contexts.length; outcomeLabels = di.getOutcomeLabels(); outcomeList = di.getOutcomeList(); predLabels = di.getPredLabels(); numPreds = predLabels.length; numOutcomes = outcomeLabels.length; display("done.\n"); display("\tNumber of Event Tokens: " + numUniqueEvents + "\n"); display("\t Number of Outcomes: " + numOutcomes + "\n"); display("\t Number of Predicates: " + numPreds + "\n"); display("Computing model parameters...\n"); MutableContext[] finalParameters = findParameters(iterations, useAverage); display("...done.\n"); /* Create and return the model *************/ return new PerceptronModel(finalParameters, predLabels, outcomeLabels); }
indexer.init(new TrainingParameters(Collections.emptyMap()), null); indexer.index(eventStream); Assert.assertEquals(3, indexer.getContexts().length); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[0]); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[1]); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[2]); Assert.assertNull(indexer.getValues()); Assert.assertEquals(5, indexer.getNumEvents());
indexer.init(new TrainingParameters(Collections.emptyMap()), null); indexer.index(eventStream); Assert.assertEquals(3, indexer.getContexts().length); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[0]); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[1]); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[2]); Assert.assertNull(indexer.getValues()); Assert.assertEquals(5, indexer.getNumEvents());
Assert.assertEquals(3, indexer.getContexts().length); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[0]); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[1]); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[2]); Assert.assertEquals(3, indexer.getValues().length); final float delta = 0.001F;
Assert.assertEquals(3, indexer.getContexts().length); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[0]); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[1]); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[2]); Assert.assertEquals(3, indexer.getValues().length); Assert.assertNull(indexer.getValues()[0]);
contexts = di.getContexts(); values = di.getValues();
public NegLogLikelihood(DataIndexer indexer) { // Get data from indexer. if (indexer instanceof OnePassRealValueDataIndexer) { this.values = indexer.getValues(); } else { this.values = null; } this.contexts = indexer.getContexts(); this.outcomeList = indexer.getOutcomeList(); this.numTimesEventsSeen = indexer.getNumTimesEventsSeen(); this.numOutcomes = indexer.getOutcomeLabels().length; this.numFeatures = indexer.getPredLabels().length; this.numContexts = this.contexts.length; this.dimension = numOutcomes * numFeatures; this.expectation = new double[numOutcomes]; this.tempSums = new double[numOutcomes]; this.gradient = new double[dimension]; }
public NegLogLikelihood(DataIndexer indexer) { // Get data from indexer. if (indexer instanceof OnePassRealValueDataIndexer) { this.values = indexer.getValues(); } else { this.values = null; } this.contexts = indexer.getContexts(); this.outcomeList = indexer.getOutcomeList(); this.numTimesEventsSeen = indexer.getNumTimesEventsSeen(); this.numOutcomes = indexer.getOutcomeLabels().length; this.numFeatures = indexer.getPredLabels().length; this.numContexts = this.contexts.length; this.dimension = numOutcomes * numFeatures; this.expectation = new double[numOutcomes]; this.tempSums = new double[numOutcomes]; this.gradient = new double[dimension]; }
/** * Evaluate the current model on training data set * @return model's training accuracy */ @Override public double evaluate(double[] parameters) { int[][] contexts = indexer.getContexts(); float[][] values = indexer.getValues(); int[] nEventsSeen = indexer.getNumTimesEventsSeen(); int[] outcomeList = indexer.getOutcomeList(); int nOutcomes = indexer.getOutcomeLabels().length; int nPredLabels = indexer.getPredLabels().length; int nCorrect = 0; int nTotalEvents = 0; for (int ei = 0; ei < contexts.length; ei++) { int[] context = contexts[ei]; float[] value = values == null ? null : values[ei]; double[] probs = new double[nOutcomes]; QNModel.eval(context, value, probs, nOutcomes, nPredLabels, parameters); int outcome = ArrayMath.argmax(probs); if (outcome == outcomeList[ei]) { nCorrect += nEventsSeen[ei]; } nTotalEvents += nEventsSeen[ei]; } return (double) nCorrect / nTotalEvents; } }
/** * Evaluate the current model on training data set * @return model's training accuracy */ @Override public double evaluate(double[] parameters) { int[][] contexts = indexer.getContexts(); float[][] values = indexer.getValues(); int[] nEventsSeen = indexer.getNumTimesEventsSeen(); int[] outcomeList = indexer.getOutcomeList(); int nOutcomes = indexer.getOutcomeLabels().length; int nPredLabels = indexer.getPredLabels().length; int nCorrect = 0; int nTotalEvents = 0; for (int ei = 0; ei < contexts.length; ei++) { int[] context = contexts[ei]; float[] value = values == null ? null : values[ei]; double[] probs = new double[nOutcomes]; QNModel.eval(context, value, probs, nOutcomes, nPredLabels, parameters); int outcome = ArrayMath.maxIdx(probs); if (outcome == outcomeList[ei]) { nCorrect += nEventsSeen[ei]; } nTotalEvents += nEventsSeen[ei]; } return (double) nCorrect / nTotalEvents; } }
public AbstractModel trainModel(int iterations, DataIndexer di, int cutoff, boolean useAverage) { display("Incorporating indexed data for training... \n"); contexts = di.getContexts(); values = di.getValues(); numTimesEventsSeen = di.getNumTimesEventsSeen(); numEvents = di.getNumEvents(); numUniqueEvents = contexts.length; outcomeLabels = di.getOutcomeLabels(); outcomeList = di.getOutcomeList(); predLabels = di.getPredLabels(); numPreds = predLabels.length; numOutcomes = outcomeLabels.length; display("done.\n"); display("\tNumber of Event Tokens: " + numUniqueEvents + "\n"); display("\t Number of Outcomes: " + numOutcomes + "\n"); display("\t Number of Predicates: " + numPreds + "\n"); display("Computing model parameters...\n"); MutableContext[] finalParameters = findParameters(iterations, useAverage); display("...done.\n"); /* Create and return the model *************/ return new PerceptronModel(finalParameters, predLabels, outcomeLabels); }
public AbstractModel trainModel(DataIndexer di) { display("Incorporating indexed data for training... \n"); contexts = di.getContexts(); values = di.getValues(); numTimesEventsSeen = di.getNumTimesEventsSeen(); numEvents = di.getNumEvents(); numUniqueEvents = contexts.length; outcomeLabels = di.getOutcomeLabels(); outcomeList = di.getOutcomeList(); predLabels = di.getPredLabels(); numPreds = predLabels.length; numOutcomes = outcomeLabels.length; display("done.\n"); display("\tNumber of Event Tokens: " + numUniqueEvents + "\n"); display("\t Number of Outcomes: " + numOutcomes + "\n"); display("\t Number of Predicates: " + numPreds + "\n"); display("Computing model parameters...\n"); MutableContext[] finalParameters = findParameters(); display("...done.\n"); /* Create and return the model ****/ return new NaiveBayesModel(finalParameters, predLabels, outcomeLabels); }
public AbstractModel trainModel(int iterations, DataIndexer di, int cutoff, boolean useAverage) { display("Incorporating indexed data for training... \n"); contexts = di.getContexts(); values = di.getValues(); numTimesEventsSeen = di.getNumTimesEventsSeen(); numEvents = di.getNumEvents(); numUniqueEvents = contexts.length; outcomeLabels = di.getOutcomeLabels(); outcomeList = di.getOutcomeList(); predLabels = di.getPredLabels(); numPreds = predLabels.length; numOutcomes = outcomeLabels.length; display("done.\n"); display("\tNumber of Event Tokens: " + numUniqueEvents + "\n"); display("\t Number of Outcomes: " + numOutcomes + "\n"); display("\t Number of Predicates: " + numPreds + "\n"); display("Computing model parameters...\n"); MutableContext[] finalParameters = findParameters(iterations, useAverage); display("...done.\n"); /* Create and return the model *************/ return new PerceptronModel(finalParameters, predLabels, outcomeLabels); }
public AbstractModel trainModel(DataIndexer di) { display("Incorporating indexed data for training... \n"); contexts = di.getContexts(); values = di.getValues(); numTimesEventsSeen = di.getNumTimesEventsSeen(); numEvents = di.getNumEvents(); numUniqueEvents = contexts.length; outcomeLabels = di.getOutcomeLabels(); outcomeList = di.getOutcomeList(); predLabels = di.getPredLabels(); numPreds = predLabels.length; numOutcomes = outcomeLabels.length; display("done.\n"); display("\tNumber of Event Tokens: " + numUniqueEvents + "\n"); display("\t Number of Outcomes: " + numOutcomes + "\n"); display("\t Number of Predicates: " + numPreds + "\n"); display("Computing model parameters...\n"); MutableContext[] finalParameters = findParameters(); display("...done.\n"); /* Create and return the model ****/ return new NaiveBayesModel(finalParameters, predLabels, outcomeLabels); }
contexts = di.getContexts(); values = di.getValues();
contexts = di.getContexts(); values = di.getValues();