private long getOffsetHelpFromMap(int ngramOrder, long key) { if (isExplicit) { return (ngramOrder >= explicitMaps.length || explicitMaps[ngramOrder] == null) ? -1 : explicitMaps[ngramOrder].getOffset(key); } return ngramOrder == 0 ? implicitUnigramMap.getOffset(key) : implicitMaps[ngramOrder - 1].getOffset(key); }
private HashNgramMap(final ValueContainer<T> values, final ConfigOptions opts, final LongArray[] numNgramsForEachWord, final boolean reversed) { super(values, opts); this.reversed = reversed; this.maxLoadFactor = opts.hashTableLoadFactor; maps = new HashMap[numNgramsForEachWord.length]; initCapacities = null; for (int ngramOrder = 0; ngramOrder < numNgramsForEachWord.length; ++ngramOrder) { maps[ngramOrder] = (ngramOrder == 0) ? new UnigramHashMap(numNgramsForEachWord[ngramOrder].size()) : new ImplicitWordHashMap( numNgramsForEachWord[ngramOrder], maxLoadFactor); values.setSizeAtLeast(maps[ngramOrder].getCapacity(), ngramOrder); } values.setMap(this); }
private HashNgramMap(final ValueContainer<T> values, final ConfigOptions opts, final LongArray[] numNgramsForEachWord, final boolean reversed) { super(values, opts); this.reversed = reversed; this.maxLoadFactor = opts.hashTableLoadFactor; this.storeSuffixOffsets = values.storeSuffixoffsets(); final int maxNgramOrder = numNgramsForEachWord.length; explicitMaps = null; isExplicit = false; implicitMaps = new ImplicitWordHashMap[maxNgramOrder - 1]; final long numWords = numNgramsForEachWord[0].size(); implicitUnigramMap = new UnigramHashMap(numWords, this); initCapacities = null; final long maxSize = getMaximumSize(numNgramsForEachWord); // a little ugly: store word ranges for all orders in the same array to increase cache locality // also, if we can, store two ints per long for cache locality final boolean fitsInInt = maxSize < Integer.MAX_VALUE; final int logicalNumRangeEntries = (maxNgramOrder - 1) * (int) numWords; final long[] wordRanges = new long[fitsInInt ? (logicalNumRangeEntries / 2 + logicalNumRangeEntries % 2) : logicalNumRangeEntries]; values.setMap(this); values.setSizeAtLeast(numWords, 0); for (int ngramOrder = 1; ngramOrder < maxNgramOrder; ++ngramOrder) { final long numNgramsForPreviousOrder = ngramOrder == 1 ? numWords : implicitMaps[ngramOrder - 2].getCapacity(); implicitMaps[ngramOrder - 1] = new ImplicitWordHashMap(numNgramsForEachWord[ngramOrder], wordRanges, ngramOrder, maxNgramOrder - 1, numNgramsForPreviousOrder, (int) numWords, this, fitsInInt, !opts.storeRankedProbBackoffs); values.setSizeAtLeast(implicitMaps[ngramOrder - 1].getCapacity(), ngramOrder); } }