/** * @param uncompressedSize * @param compressedLongArray * @param keyBits * @param valueBits */ private void logCompressionInfo(final long uncompressedSize, final LongArray compressedLongArray, final long keyBits, final long valueBits) { final double keyAvg = (double) keyBits / uncompressedSize; Logger.logss("Key bits " + keyAvg); final double valueAvg = (double) valueBits / uncompressedSize; Logger.logss("Value bits " + valueAvg); final double avg = 64 * (double) compressedLongArray.size() / uncompressedSize; Logger.logss("Compressed bits " + avg); totalKeyBitsFinal += keyBits; totalValueBitsFinal += valueBits; totalBitsFinal += compressedLongArray.size(); totalSizeFinal += uncompressedSize; Logger.logss("Total key bits " + totalKeyBitsFinal / totalSizeFinal); Logger.logss("Total value bits " + totalValueBitsFinal / totalSizeFinal); Logger.logss("Total bits " + 64.0 * totalBitsFinal / totalSizeFinal); }
/** * @param uncompressedSize * @param compressedLongArray * @param keyBits * @param valueBits */ private void logCompressionInfo(final long uncompressedSize, final LongArray compressedLongArray, final long keyBits, final long valueBits) { final double keyAvg = (double) keyBits / uncompressedSize; Logger.logss("Key bits " + keyAvg); final double valueAvg = (double) valueBits / uncompressedSize; Logger.logss("Value bits " + valueAvg); final double avg = 64 * (double) compressedLongArray.size() / uncompressedSize; Logger.logss("Compressed bits " + avg); totalKeyBitsFinal += keyBits; totalValueBitsFinal += valueBits; totalBitsFinal += compressedLongArray.size(); totalSizeFinal += uncompressedSize; Logger.logss("Total key bits " + totalKeyBitsFinal / totalSizeFinal); Logger.logss("Total value bits " + totalValueBitsFinal / totalSizeFinal); Logger.logss("Total bits " + 64.0 * totalBitsFinal / totalSizeFinal); }
@Override public void cleanup() { Logger.startTrack("Cleaning up values"); Logger.logss("Found " + valueCounter.size() + " unique counts"); Logger.endTrack(); }
public ImplicitWordHashMap(final LongArray numNgramsForEachWord, final double loadFactor) { final long numWords = numNgramsForEachWord.size(); wordRanges = new long[(int) numWords]; final long totalNumNgrams = setWordRanges(numNgramsForEachWord, loadFactor, numWords); keys = LongArray.StaticMethods.newLongArray(totalNumNgrams, totalNumNgrams, totalNumNgrams); Logger.logss("No word key size " + totalNumNgrams); keys.fill(EMPTY_KEY, totalNumNgrams); numFilled = 0; }
@Override public void trim() { for (int ngramOrder = 0; ngramOrder < maps.length; ++ngramOrder) { if (maps[ngramOrder] == null) break; values.trimAfterNgram(ngramOrder, maps[ngramOrder].getCapacity()); Logger.logss("Load factor for " + (ngramOrder + 1) + ": " + maps[ngramOrder].getLoadFactor()); } values.trim(); }
@Override public void cleanup() { Logger.startTrack("Cleaning up values"); valueIndexer = new Indexer<V>(); for (final Entry<V, Double> entry : valueCounter.getEntriesSortedByDecreasingCount()) { valueIndexer.add(entry.getKey()); } Logger.logss("Found " + valueIndexer.size() + " unique counts"); valueCounter = null; Logger.endTrack(); }
out.println("\\" + (ngramOrder + 1) + "-grams:"); int line = 0; Logger.logss("On order " + (ngramOrder + 1)); int linenum = 0; for (Entry<KneserNeyCounts> entry : ngrams.getNgramsForOrder(ngramOrder)) {
@Override public void trim() { for (int ngramOrder = 0; ngramOrder < getMaxNgramOrder(); ++ngramOrder) { final HashMap currMap = getMap(ngramOrder); if (currMap == null) break; values.trimAfterNgram(ngramOrder, currMap.getCapacity()); Logger.logss("Load factor for " + (ngramOrder + 1) + ": " + currMap.getLoadFactor()); } values.trim(); }
callback.handleNgramOrderStarted(ngramOrder + 1); Logger.logss("On order " + (ngramOrder + 1)); int linenum = 0; for (final Entry<KneserNeyCounts> entry : ngrams.getNgramsForOrder(ngramOrder)) {
Logger.logss("Storing count indices using " + valueWidth + " bits."); Logger.endTrack();
public UncompressedProbBackoffValueContainer(final LongToIntHashMap countCounter, final int valueRadix, final boolean storePrefixes, long[] numNgramsForEachOrder) { super(valueRadix, storePrefixes, numNgramsForEachOrder); Logger.startTrack("Storing values"); final long defaultVal = getDefaultVal().asLong(); final boolean hasDefaultVal = countCounter.get(defaultVal, -1) >= 0; probsAndBackoffsForRank = new long[(countCounter.size() + (hasDefaultVal ? 0 : 1))]; countIndexer = new LongToIntHashMap(); int k = 0; for (final Entry pair : countCounter.getObjectsSortedByValue(true)) { countIndexer.put(pair.key, countIndexer.size()); probsAndBackoffsForRank[k++] = pair.key; if (countIndexer.size() == defaultValRank && !hasDefaultVal) { countIndexer.put(defaultVal, countIndexer.size()); probsAndBackoffsForRank[k++] = defaultVal; } } if (countIndexer.size() < defaultValRank && !hasDefaultVal) { countIndexer.put(defaultVal, countIndexer.size()); probsAndBackoffsForRank[k++] = defaultVal; } valueWidth = CustomWidthArray.numBitsNeeded(countIndexer.size()); Logger.logss("Storing count indices using " + valueWidth + " bits."); Logger.endTrack(); }