public SparseIntArray[][] readAll(int W) throws IOException { final ArrayList<SparseIntArray[]> sparseArrays = new ArrayList<SparseIntArray[]>(); SparseIntArray[] sa; while ((sa = nextFreqPair(W)) != null) { sparseArrays.add(sa); } return sparseArrays.toArray(new SparseIntArray[sparseArrays.size()][]); }
public Object2IntMap<NGram>[][] readAllNGrams(int N) throws IOException { final ArrayList<Object2IntMap<NGram>[]> sparseArrays = new ArrayList<Object2IntMap<NGram>[]>(); Object2IntMap<NGram>[] sa; while ((sa = nextNGramPair(N)) != null) { sparseArrays.add(sa); } return sparseArrays.toArray(new Object2IntMap[sparseArrays.size()][]); }
public SparseIntArray[] nextFreqPair(int W) throws IOException { final int[][] pair = nextPair(); if (pair == null) { return null; } else { return new SparseIntArray[]{ SparseIntArray.histogram(pair[0], W), SparseIntArray.histogram(pair[1], W) }; } }
public Object2IntMap<NGram>[] nextNGramPair(int N) throws IOException { final int[][] pair = nextPair(); if (pair == null) { return null; } else { final NGramCarousel carousel = new NGramCarousel(N); final Object2IntRBTreeMap<NGram>[] ngramPair = new Object2IntRBTreeMap[]{ new Object2IntRBTreeMap(), new Object2IntRBTreeMap() }; for (int l = 0; l < 2; l++) { for (int i = 0; i < pair[l].length; i++) { carousel.offer(pair[l][i]); for (int n = 1; n <= carousel.maxNGram(); n++) { final NGram ng = carousel.ngram(n); if (ngramPair[l].containsKey(ng)) { ngramPair[l].put(ng, ngramPair[l].getInt(ng) + 1); } else { ngramPair[l].put(ng, 1); } } } } return ngramPair; } }
@Override public SimilarityMetric makeMetric(ParallelBinarizedReader data, int W) throws IOException { final ArrayList<SparseIntArray[]> x = new ArrayList<SparseIntArray[]>(); while (true) { final SparseIntArray[] nextFreqPair = data.nextFreqPair(W); if (nextFreqPair == null) { break; } else { x.add(nextFreqPair); } } return new Salience(x.toArray(new SparseIntArray[x.size()][]), W, 0.5); }
@Override public SimilarityMetric makeMetric(ParallelBinarizedReader data, int W) throws IOException { final ArrayList<SparseIntArray[]> x = new ArrayList<SparseIntArray[]>(); while (true) { final SparseIntArray[] nextFreqPair = data.nextFreqPair(W); if(nextFreqPair == null) { break; } else { x.add(nextFreqPair); } } return new BetaLMImpl(x.toArray(new SparseIntArray[x.size()][]), W); }