private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { stream.defaultReadObject(); Set<UnaryRule> allRules = Generics.newHashSet(coreRules.keySet()); init(); for (UnaryRule ur : allRules) { addRule(ur); } purgeRules(); }
/** * Populates data in this UnaryGrammar from a character stream. * * @param in The Reader the grammar is read from. * @throws IOException If there is a reading problem */ public void readData(BufferedReader in) throws IOException { String line; int lineNum = 1; // all lines have one rule per line line = in.readLine(); while (line != null && line.length() > 0) { try { addRule(new UnaryRule(line, index)); } catch (Exception e) { throw new IOException("Error on line " + lineNum); } lineNum++; line = in.readLine(); } purgeRules(); }
newUG.addRule(unaryRule);
@Override public Pair<UnaryGrammar,BinaryGrammar> formResult() { stateIndex.addToIndex(Lexicon.BOUNDARY_TAG); BinaryGrammar bg = new BinaryGrammar(stateIndex); UnaryGrammar ug = new UnaryGrammar(stateIndex); // add unaries for (UnaryRule ur : unaryRules) { ur.score = (float) Math.log(unaryRuleCounter.getCount(ur) / symbolCounter.getCount(stateIndex.get(ur.parent))); if (op.trainOptions.compactGrammar() >= 4) { ur.score = (float) unaryRuleCounter.getCount(ur); } ug.addRule(ur); } // add binaries for (BinaryRule br : binaryRules) { br.score = (float) Math.log((binaryRuleCounter.getCount(br) - op.trainOptions.ruleDiscount) / symbolCounter.getCount(stateIndex.get(br.parent))); if (op.trainOptions.compactGrammar() >= 4) { br.score = (float) binaryRuleCounter.getCount(br); } bg.addRule(br); } return new Pair<>(ug, bg); }
rule.score = (float) Math.log(rule.score / count); ug.addRule(rule); numRules++;
double score = betas[i][j] - Math.log(stateTotal[i]); UnaryRule ur = new UnaryRule(parentIndex, childIndex, score); ug.addRule(ur);
private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { stream.defaultReadObject(); Set<UnaryRule> allRules = new HashSet<UnaryRule>(coreRules.keySet()); init(); for (UnaryRule ur : allRules) { addRule(ur); } purgeRules(); }
private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { stream.defaultReadObject(); Set<UnaryRule> allRules = Generics.newHashSet(coreRules.keySet()); init(); for (UnaryRule ur : allRules) { addRule(ur); } purgeRules(); }
private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { stream.defaultReadObject(); Set<UnaryRule> allRules = Generics.newHashSet(coreRules.keySet()); init(); for (UnaryRule ur : allRules) { addRule(ur); } purgeRules(); }
/** * Populates data in this UnaryGrammar from a character stream. * * @param in The Reader the grammar is read from. * @throws IOException If there is a reading problem */ public void readData(BufferedReader in) throws IOException { String line; int lineNum = 1; // all lines have one rule per line line = in.readLine(); while (line != null && line.length() > 0) { try { addRule(new UnaryRule(line, index)); } catch (Exception e) { throw new IOException("Error on line " + lineNum); } lineNum++; line = in.readLine(); } purgeRules(); }
private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { stream.defaultReadObject(); Set<UnaryRule> allRules = Generics.newHashSet(coreRules.keySet()); init(); for (UnaryRule ur : allRules) { addRule(ur); } purgeRules(); }
/** * Populates data in this UnaryGrammar from a character stream. * * @param in The Reader the grammar is read from. * @throws IOException If there is a reading problem */ public void readData(BufferedReader in) throws IOException { String line; int lineNum = 1; // all lines have one rule per line line = in.readLine(); while (line != null && line.length() > 0) { try { addRule(new UnaryRule(line, index)); } catch (Exception e) { throw new IOException("Error on line " + lineNum); } lineNum++; line = in.readLine(); } purgeRules(); }
/** * Populates data in this UnaryGrammar from a character stream. * * @param in The Reader the grammar is read from. * @throws IOException If there is a reading problem */ public void readData(BufferedReader in) throws IOException { String line; int lineNum = 1; // all lines have one rule per line line = in.readLine(); while (line != null && line.length() > 0) { try { addRule(new UnaryRule(line, index)); } catch (Exception e) { throw new IOException("Error on line " + lineNum); } lineNum++; line = in.readLine(); } purgeRules(); }
/** * Populates data in this UnaryGrammar from a character stream. * * @param in The Reader the grammar is read from. * @throws IOException If there is a reading problem */ public void readData(BufferedReader in) throws IOException { String line; int lineNum = 1; Numberer n = Numberer.getGlobalNumberer(stateSpace); // all lines have one rule per line line = in.readLine(); while (line != null && line.length() > 0) { try { addRule(new UnaryRule(line, n)); } catch (Exception e) { throw new IOException("Error on line " + lineNum); } lineNum++; line = in.readLine(); } purgeRules(); }
@Override public Pair<UnaryGrammar,BinaryGrammar> formResult() { stateNumberer.number(Lexicon.BOUNDARY_TAG); BinaryGrammar bg = new BinaryGrammar(stateNumberer.total()); UnaryGrammar ug = new UnaryGrammar(stateNumberer.total()); // add unaries for (Iterator<Rule> uI = unaryRules.iterator(); uI.hasNext();) { UnaryRule ur = (UnaryRule) uI.next(); ur.score = (float) Math.log(ruleCounter.getCount(ur) / symbolCounter.getCount((String)stateNumberer.object(ur.parent))); if (Train.compactGrammar() >= 4) { ur.score = (float) ruleCounter.getCount(ur); } ug.addRule(ur); } // add binaries for (Iterator<Rule> bI = binaryRules.iterator(); bI.hasNext();) { BinaryRule br = (BinaryRule) bI.next(); br.score = (float) Math.log((ruleCounter.getCount(br) - Train.ruleDiscount) / symbolCounter.getCount((String)stateNumberer.object(br.parent))); if (Train.compactGrammar() >= 4) { br.score = (float) ruleCounter.getCount(br); } bg.addRule(br); } return new Pair<UnaryGrammar,BinaryGrammar>(ug, bg); }
@Override public Pair<UnaryGrammar,BinaryGrammar> formResult() { stateIndex.addToIndex(Lexicon.BOUNDARY_TAG); BinaryGrammar bg = new BinaryGrammar(stateIndex); UnaryGrammar ug = new UnaryGrammar(stateIndex); // add unaries for (UnaryRule ur : unaryRules) { ur.score = (float) Math.log(unaryRuleCounter.getCount(ur) / symbolCounter.getCount(stateIndex.get(ur.parent))); if (op.trainOptions.compactGrammar() >= 4) { ur.score = (float) unaryRuleCounter.getCount(ur); } ug.addRule(ur); } // add binaries for (BinaryRule br : binaryRules) { br.score = (float) Math.log((binaryRuleCounter.getCount(br) - op.trainOptions.ruleDiscount) / symbolCounter.getCount(stateIndex.get(br.parent))); if (op.trainOptions.compactGrammar() >= 4) { br.score = (float) binaryRuleCounter.getCount(br); } bg.addRule(br); } return new Pair<>(ug, bg); }
@Override public Pair<UnaryGrammar,BinaryGrammar> formResult() { stateIndex.addToIndex(Lexicon.BOUNDARY_TAG); BinaryGrammar bg = new BinaryGrammar(stateIndex); UnaryGrammar ug = new UnaryGrammar(stateIndex); // add unaries for (UnaryRule ur : unaryRules) { ur.score = (float) Math.log(unaryRuleCounter.getCount(ur) / symbolCounter.getCount(stateIndex.get(ur.parent))); if (op.trainOptions.compactGrammar() >= 4) { ur.score = (float) unaryRuleCounter.getCount(ur); } ug.addRule(ur); } // add binaries for (BinaryRule br : binaryRules) { br.score = (float) Math.log((binaryRuleCounter.getCount(br) - op.trainOptions.ruleDiscount) / symbolCounter.getCount(stateIndex.get(br.parent))); if (op.trainOptions.compactGrammar() >= 4) { br.score = (float) binaryRuleCounter.getCount(br); } bg.addRule(br); } return new Pair<>(ug, bg); }
@Override public Pair<UnaryGrammar,BinaryGrammar> formResult() { stateIndex.indexOf(Lexicon.BOUNDARY_TAG, true); BinaryGrammar bg = new BinaryGrammar(stateIndex); UnaryGrammar ug = new UnaryGrammar(stateIndex); // add unaries for (UnaryRule ur : unaryRules) { ur.score = (float) Math.log(unaryRuleCounter.getCount(ur) / symbolCounter.getCount(stateIndex.get(ur.parent))); if (op.trainOptions.compactGrammar() >= 4) { ur.score = (float) unaryRuleCounter.getCount(ur); } ug.addRule(ur); } // add binaries for (BinaryRule br : binaryRules) { br.score = (float) Math.log((binaryRuleCounter.getCount(br) - op.trainOptions.ruleDiscount) / symbolCounter.getCount(stateIndex.get(br.parent))); if (op.trainOptions.compactGrammar() >= 4) { br.score = (float) binaryRuleCounter.getCount(br); } bg.addRule(br); } return new Pair<UnaryGrammar,BinaryGrammar>(ug, bg); }
double score = betas[i][j] - Math.log(stateTotal[i]); UnaryRule ur = new UnaryRule(parentIndex, childIndex, score); ug.addRule(ur);