evaluatorList.add(new MemoryEvaluator()); if (flags.evaluateTrain) { CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<>("Train set", this); List<Triple<int[][][], int[], double[][][]>> trainDataAndLabels = new ArrayList<>(); int[][][][] data = dataAndLabelsAndFeatureVals.first(); crfEvaluator.setTestData(docs, trainDataAndLabels); if (flags.evalCmd.length() > 0) crfEvaluator.setEvalCmd(flags.evalCmd); evaluatorList.add(crfEvaluator); CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<>("Test set (" + flags.testFile + ")", this); ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(flags.testFile, readerAndWriter); List<List<IN>> testDocs = new ArrayList<>(testObjBank); List<Triple<int[][][], int[], double[][][]>> testDataAndLabels = documentsToDataAndLabelsList(testDocs); crfEvaluator.setTestData(testDocs, testDataAndLabels); if ( ! flags.evalCmd.isEmpty()) { crfEvaluator.setEvalCmd(flags.evalCmd); String[] testFiles = flags.testFiles.split(","); for (String testFile : testFiles) { CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<>("Test set (" + testFile + ')', this); ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(testFile, readerAndWriter); List<Triple<int[][][], int[], double[][][]>> testDataAndLabels = documentsToDataAndLabelsList(testObjBank); crfEvaluator.setTestData(testObjBank, testDataAndLabels); if ( ! flags.evalCmd.isEmpty()) { crfEvaluator.setEvalCmd(flags.evalCmd);
@Override public double evaluate(double[] x) { double score; // initialized below setValues(x); if (getCmd() != null) { evaluateCmd(getCmd()); score = interpretCmdOutput(); } else { try { // TODO: Classify in memory instead of writing to tmp file File f = File.createTempFile("CRFClassifierEvaluator","txt"); f.deleteOnExit(); OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(f)); PrintWriter pw = IOUtils.encodedOutputStreamPrintWriter(outputStream, null, true); classifier.classifyAndWriteAnswers(data, featurizedData, pw, classifier.makeReaderAndWriter()); outputStream.close(); BufferedReader br = new BufferedReader(new FileReader(f)); MultiClassChunkEvalStats stats = new MultiClassChunkEvalStats("O"); score = stats.score(br, "\t"); log.info(stats.getConllEvalString()); f.delete(); } catch (Exception ex) { throw new RuntimeException(ex); } } return score; }
private double interpretCmdOutput() { String output = getOutput(); String[] parts = output.split("\\s+"); int fScoreIndex = 0; for (; fScoreIndex < parts.length; fScoreIndex++) if (parts[fScoreIndex].equals("FB1:")) break; fScoreIndex += 1; if (fScoreIndex < parts.length) return Double.parseDouble(parts[fScoreIndex]); else { log.error("in CRFClassifierEvaluator.interpretCmdOutput(), cannot find FB1 score in output:\n"+output); return -1; } }
public double evaluate(double[] x) { double score = 0; setValues(x); if (getCmd() != null) { evaluateCmd(getCmd()); } else { try { // TODO: Classify in memory instead of writing to tmp file File f = File.createTempFile("CRFClassifierEvaluator","txt"); f.deleteOnExit(); OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(f)); classifier.classifyAndWriteAnswers(data, featurizedData, outputStream, classifier.makeReaderAndWriter()); outputStream.close(); BufferedReader br = new BufferedReader(new FileReader(f)); MultiClassChunkEvalStats stats = new MultiClassChunkEvalStats("O"); score = stats.score(br, "\t"); System.err.println(stats.getConllEvalString()); f.delete(); } catch (Exception ex) { throw new RuntimeException(ex); } } return score; }
evaluatorList.add(new MemoryEvaluator()); if (flags.evaluateTrain) { CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Train set", this); List<Pair<int[][][],int[]>> trainDataAndLabels = new ArrayList<Pair<int[][][],int[]>>(); for (int j = 0; j < dataAndLabels.first().length; j++) { crfEvaluator.setTestData(docs, trainDataAndLabels); crfEvaluator.setEvalCmd(flags.evalCmd); evaluatorList.add(crfEvaluator); CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Test set (" + flags.testFile + ")", this); ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(flags.testFile, readerAndWriter); List<Pair<int[][][],int[]>> testDataAndLabels = documentsToDataAndLabelsList(testObjBank); crfEvaluator.setTestData(testObjBank, testDataAndLabels); crfEvaluator.setEvalCmd(flags.evalCmd); evaluatorList.add(crfEvaluator); for (String testFile: testFiles) { CRFClassifierEvaluator<IN> crfEvaluator = crfEvaluator = new CRFClassifierEvaluator<IN>("Test set (" + testFile + ")", this); ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(testFile, readerAndWriter); List<Pair<int[][][],int[]>> testDataAndLabels = documentsToDataAndLabelsList(testObjBank); crfEvaluator.setTestData(testObjBank, testDataAndLabels); crfEvaluator.setEvalCmd(flags.evalCmd); evaluatorList.add(crfEvaluator); for (Evaluator eval: evaluators) { if (eval instanceof CRFClassifierEvaluator) {
public CRFClassifierEvaluator(String description, CRFClassifier<IN> classifier, Collection<List<IN>> data, List<Triple<int[][][], int[], double[][][]>> featurizedData) { this.description = description; this.classifier = classifier; this.data = data; this.featurizedData = featurizedData; cmd = getCmd(cmdStr); saveOutput = true; }
/** * Set the evaluation command (set to null to skip evaluation using command line) * @param evalCmd */ public void setEvalCmd(String evalCmd) { log.info("setEvalCmd to " + evalCmd); this.cmdStr = evalCmd; if (cmdStr != null) { cmdStr = cmdStr.trim(); if (cmdStr.isEmpty()) { cmdStr = null; } } cmd = getCmd(cmdStr); }
public double evaluate(double[] x) { double score = 0; setValues(x); if (getCmd() != null) { evaluateCmd(getCmd()); score = interpretCmdOutput(); } else { try { // TODO: Classify in memory instead of writing to tmp file File f = File.createTempFile("CRFClassifierEvaluator","txt"); f.deleteOnExit(); OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(f)); PrintWriter pw = IOUtils.encodedOutputStreamPrintWriter(outputStream, null, true); classifier.classifyAndWriteAnswers(data, featurizedData, pw, classifier.makeReaderAndWriter()); outputStream.close(); BufferedReader br = new BufferedReader(new FileReader(f)); MultiClassChunkEvalStats stats = new MultiClassChunkEvalStats("O"); score = stats.score(br, "\t"); System.err.println(stats.getConllEvalString()); f.delete(); } catch (Exception ex) { throw new RuntimeException(ex); } } return score; }
evaluatorList.add(new MemoryEvaluator()); if (flags.evaluateTrain) { CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Train set", this); List<Triple<int[][][], int[], double[][][]>> trainDataAndLabels = new ArrayList<Triple<int[][][], int[], double[][][]>>(); int[][][][] data = dataAndLabelsAndFeatureVals.first(); crfEvaluator.setTestData(docs, trainDataAndLabels); if (flags.evalCmd.length() > 0) crfEvaluator.setEvalCmd(flags.evalCmd); evaluatorList.add(crfEvaluator); CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Test set (" + flags.testFile + ")", this); ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(flags.testFile, readerAndWriter); crfEvaluator.setTestData(testDocs, testDataAndLabels); if (flags.evalCmd.length() > 0) crfEvaluator.setEvalCmd(flags.evalCmd); evaluatorList.add(crfEvaluator); String[] testFiles = flags.testFiles.split(","); for (String testFile : testFiles) { CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<IN>("Test set (" + testFile + ")", this); ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(testFile, readerAndWriter); List<Triple<int[][][], int[], double[][][]>> testDataAndLabels = documentsToDataAndLabelsList(testObjBank); crfEvaluator.setTestData(testObjBank, testDataAndLabels); if (flags.evalCmd.length() > 0) crfEvaluator.setEvalCmd(flags.evalCmd);
public CRFClassifierEvaluator(String description, CRFClassifier<IN> classifier, Collection<List<IN>> data, List<Triple<int[][][], int[], double[][][]>> featurizedData) { this.description = description; this.classifier = classifier; this.data = data; this.featurizedData = featurizedData; cmd = getCmd(cmdStr); saveOutput = true; }
private double interpretCmdOutput() { String output = getOutput(); String[] parts = output.split("\\s+"); int fScoreIndex = 0; for (; fScoreIndex < parts.length; fScoreIndex++) if (parts[fScoreIndex].equals("FB1:")) break; fScoreIndex += 1; if (fScoreIndex < parts.length) return Double.parseDouble(parts[fScoreIndex]); else { System.err.println("ERROR in CRFClassifierEvaluator.interpretCmdOutput(), cannot find FB1 score in output:\n"+output); return -1; } }
@Override public double evaluate(double[] x) { double score; // initialized below setValues(x); if (getCmd() != null) { evaluateCmd(getCmd()); score = interpretCmdOutput(); } else { try { // TODO: Classify in memory instead of writing to tmp file File f = File.createTempFile("CRFClassifierEvaluator","txt"); f.deleteOnExit(); OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(f)); PrintWriter pw = IOUtils.encodedOutputStreamPrintWriter(outputStream, null, true); classifier.classifyAndWriteAnswers(data, featurizedData, pw, classifier.makeReaderAndWriter()); outputStream.close(); BufferedReader br = new BufferedReader(new FileReader(f)); MultiClassChunkEvalStats stats = new MultiClassChunkEvalStats("O"); score = stats.score(br, "\t"); log.info(stats.getConllEvalString()); f.delete(); } catch (Exception ex) { throw new RuntimeException(ex); } } return score; }
evaluatorList.add(new MemoryEvaluator()); if (flags.evaluateTrain) { CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<>("Train set", this); List<Triple<int[][][], int[], double[][][]>> trainDataAndLabels = new ArrayList<>(); int[][][][] data = dataAndLabelsAndFeatureVals.first(); crfEvaluator.setTestData(docs, trainDataAndLabels); if (flags.evalCmd.length() > 0) crfEvaluator.setEvalCmd(flags.evalCmd); evaluatorList.add(crfEvaluator); CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<>("Test set (" + flags.testFile + ")", this); ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(flags.testFile, readerAndWriter); List<List<IN>> testDocs = new ArrayList<>(testObjBank); List<Triple<int[][][], int[], double[][][]>> testDataAndLabels = documentsToDataAndLabelsList(testDocs); crfEvaluator.setTestData(testDocs, testDataAndLabels); if ( ! flags.evalCmd.isEmpty()) { crfEvaluator.setEvalCmd(flags.evalCmd); String[] testFiles = flags.testFiles.split(","); for (String testFile : testFiles) { CRFClassifierEvaluator<IN> crfEvaluator = new CRFClassifierEvaluator<>("Test set (" + testFile + ')', this); ObjectBank<List<IN>> testObjBank = makeObjectBankFromFile(testFile, readerAndWriter); List<Triple<int[][][], int[], double[][][]>> testDataAndLabels = documentsToDataAndLabelsList(testObjBank); crfEvaluator.setTestData(testObjBank, testDataAndLabels); if ( ! flags.evalCmd.isEmpty()) { crfEvaluator.setEvalCmd(flags.evalCmd);
public CRFClassifierEvaluator(String description, CRFClassifier<IN> classifier, Collection<List<IN>> data, List<Triple<int[][][], int[], double[][][]>> featurizedData) { this.description = description; this.classifier = classifier; this.data = data; this.featurizedData = featurizedData; cmd = getCmd(cmdStr); saveOutput = true; }
private double interpretCmdOutput() { String output = getOutput(); String[] parts = output.split("\\s+"); int fScoreIndex = 0; for (; fScoreIndex < parts.length; fScoreIndex++) if (parts[fScoreIndex].equals("FB1:")) break; fScoreIndex += 1; if (fScoreIndex < parts.length) return Double.parseDouble(parts[fScoreIndex]); else { log.error("in CRFClassifierEvaluator.interpretCmdOutput(), cannot find FB1 score in output:\n"+output); return -1; } }
public CRFClassifierEvaluator(String description, CRFClassifier<IN> classifier, CRFLogConditionalObjectiveFunction func, Collection<List<IN>> data, List<Pair<int[][][], int[]>> featurizedData) { this.description = description; this.classifier = classifier; this.func = func; this.data = data; this.featurizedData = featurizedData; cmd = getCmd(cmdStr); }
/** * Set the evaluation command (set to null to skip evaluation using command line) * @param evalCmd */ public void setEvalCmd(String evalCmd) { this.cmdStr = evalCmd; if (cmdStr != null) { cmdStr = cmdStr.trim(); if (cmdStr.length() == 0) { cmdStr = null; } } cmd = getCmd(cmdStr); }
/** * Set the evaluation command (set to null to skip evaluation using command line) * @param evalCmd */ public void setEvalCmd(String evalCmd) { System.err.println("setEvalCmd to " + evalCmd); this.cmdStr = evalCmd; if (cmdStr != null) { cmdStr = cmdStr.trim(); if (cmdStr.length() == 0) { cmdStr = null; } } cmd = getCmd(cmdStr); }
/** * Set the evaluation command (set to null to skip evaluation using command line) * @param evalCmd */ public void setEvalCmd(String evalCmd) { log.info("setEvalCmd to " + evalCmd); this.cmdStr = evalCmd; if (cmdStr != null) { cmdStr = cmdStr.trim(); if (cmdStr.isEmpty()) { cmdStr = null; } } cmd = getCmd(cmdStr); }