UnaryGrammar ug = new UnaryGrammar(newStateIndex); BinaryGrammar bg = new BinaryGrammar(newStateIndex); for (UnaryRule rule : unaryRules) { rule.score = (float) Math.log(rule.score / count); ug.addRule(rule); numRules++; ug.purgeRules(); bg.splitRules(); return new Pair<>(ug, bg);
public final void addRule(UnaryRule ur) { // add rules' closure closeRulesUnderMax(ur); coreRules.put(ur, ur); rulesWithParent[ur.parent].add(ur); rulesWithChild[ur.child].add(ur); }
for (Iterator<UnaryRule> rI = ruleIterator(); rI.hasNext(); ) { out.println(rI.next().toString(index)); for (Iterator<UnaryRule> rI = closedRuleIterator(); rI.hasNext(); ) { out.println(rI.next().toString(index)); for (int i = 0; i < numStates; i++) { out.println(index.get(i)); for (Iterator<UnaryRule> rI = ruleIteratorByParent(i); rI.hasNext(); ) { out.print(" "); out.println(rI.next().toString(index)); for (int i = 0; i < numStates; i++) { out.println(index.get(i)); for (Iterator<UnaryRule> rI = closedRuleIteratorByParent(i); rI.hasNext(); ) { out.print(" "); out.println(rI.next().toString(index));
private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { stream.defaultReadObject(); Set<UnaryRule> allRules = Generics.newHashSet(coreRules.keySet()); init(); for (UnaryRule ur : allRules) { addRule(ur); } purgeRules(); }
/** * Populates data in this UnaryGrammar from a character stream. * * @param in The Reader the grammar is read from. * @throws IOException If there is a reading problem */ public void readData(BufferedReader in) throws IOException { String line; int lineNum = 1; // all lines have one rule per line line = in.readLine(); while (line != null && line.length() > 0) { try { addRule(new UnaryRule(line, index)); } catch (Exception e) { throw new IOException("Error on line " + lineNum); } lineNum++; line = in.readLine(); } purgeRules(); }
@Override public Pair<UnaryGrammar,BinaryGrammar> formResult() { stateIndex.addToIndex(Lexicon.BOUNDARY_TAG); BinaryGrammar bg = new BinaryGrammar(stateIndex); UnaryGrammar ug = new UnaryGrammar(stateIndex); // add unaries for (UnaryRule ur : unaryRules) { ur.score = (float) Math.log(unaryRuleCounter.getCount(ur) / symbolCounter.getCount(stateIndex.get(ur.parent))); if (op.trainOptions.compactGrammar() >= 4) { ur.score = (float) unaryRuleCounter.getCount(ur); } ug.addRule(ur); } // add binaries for (BinaryRule br : binaryRules) { br.score = (float) Math.log((binaryRuleCounter.getCount(br) - op.trainOptions.ruleDiscount) / symbolCounter.getCount(stateIndex.get(br.parent))); if (op.trainOptions.compactGrammar() >= 4) { br.score = (float) binaryRuleCounter.getCount(br); } bg.addRule(br); } return new Pair<>(ug, bg); }
UnaryGrammar ug = grammar.first; List<UnaryRule> unaryRules = new ArrayList<>(); for (UnaryRule rule : ug.rules()) { rule.parent = translate(rule.parent, oldIndex, newIndex); rule.child = translate(rule.child, oldIndex, newIndex); UnaryGrammar newUG = new UnaryGrammar(newIndex); for (UnaryRule unaryRule : unaryRules) { newUG.addRule(unaryRule); newUG.purgeRules();
UnaryGrammar ug = new UnaryGrammar(stateIndex); ug.readData(in);
continue; UnaryRule[] unaries = ug.closedRulesByChild(state); for (UnaryRule ur : unaries) { int parentState = ur.parent;
System.out.println("All states size: " + parser.stateIndex.size()); System.out.println("Unary grammar size: " + parser.ug.numRules()); System.out.println("Binary grammar size: " + parser.bg.numRules());
/** Remove A -> A UnaryRules from bestRulesUnderMax. */ public final void purgeRules() { Map<UnaryRule,UnaryRule> bR = Generics.newHashMap(); for (UnaryRule ur : bestRulesUnderMax.keySet()) { if (ur.parent != ur.child) { bR.put(ur, ur); } else { closedRulesWithParent[ur.parent].remove(ur); closedRulesWithChild[ur.child].remove(ur); } } bestRulesUnderMax = bR; makeCRArrays(); }
public UnaryGrammar(Index<String> stateIndex) { this.index = stateIndex; init(); }
continue; UnaryRule[] rules = ug.closedRulesByParent(s); for (UnaryRule ur : rules) { float pS = ur.score;
private void readObject(ObjectInputStream stream) throws IOException, ClassNotFoundException { stream.defaultReadObject(); Set<UnaryRule> allRules = new HashSet<UnaryRule>(coreRules.keySet()); init(); for (UnaryRule ur : allRules) { addRule(ur); } purgeRules(); }
UnaryGrammar ug = new UnaryGrammar(stateIndex); for (String parent : tempUnaryBetas.firstKeySet()) { int parentStates = getStateSplitCount(parent); double score = betas[i][j] - Math.log(stateTotal[i]); UnaryRule ur = new UnaryRule(parentIndex, childIndex, score); ug.addRule(ur);
/** * Populates data in this UnaryGrammar from a character stream. * * @param in The Reader the grammar is read from. * @throws IOException If there is a reading problem */ public void readData(BufferedReader in) throws IOException { String line; int lineNum = 1; // all lines have one rule per line line = in.readLine(); while (line != null && line.length() > 0) { try { addRule(new UnaryRule(line, index)); } catch (Exception e) { throw new IOException("Error on line " + lineNum); } lineNum++; line = in.readLine(); } purgeRules(); }
line = in.readLine(); confirmBeginBlock(textFileOrUrl, line); UnaryGrammar ug = new UnaryGrammar(op.numStates); ug.readData(in); System.err.print("."); line = in.readLine();
UnaryRule[] unaries = ug.closedRulesByChild(state); for (UnaryRule ur : unaries) {
lp.tagIndex.size() + '\t' + lp.wordIndex.size() + '\t' + (pparser != null ? lp.ug.numRules(): "") + '\t' + (pparser != null ? lp.bg.numRules(): "") + '\t' + lexNumRules);
public UnaryRule[] closedRulesByParent(int state) { if (closedRulesWithP == null) { makeCRArrays(); } if (state >= closedRulesWithP.length) { return EMPTY_UNARY_RULE_ARRAY; } return closedRulesWithP[state]; }