/** * Getter for allocated byte array to which trie is mapped * @return byte[] with ngram trie */ public byte[] getMem() { return bitArr.getArr(); }
/** * Reads encoded backoff from provided trie bit array and decodes it into actual value * for specific ngram * @param bitArr - trie bit array * @param memPtr - memory pointer for specific ngram order * @param bitOffset - offset from memPtr that is calculated according to ngram index * @param orderMinusTwo - order of ngram minus two * @return backoffs of ngram */ public float readBackoff(NgramTrieBitarr bitArr, int memPtr, int bitOffset, int orderMinusTwo) { switch (quantType) { case NO_QUANT: bitOffset += 31; return bitArr.readFloat(memPtr, bitOffset); case QUANT_16: int tableIdx = orderMinusTwo * 2 + 1; return binsDecode(tableIdx, bitArr.readInt(memPtr, bitOffset, probMask)); //TODO implement different quantization stages default: throw new Error("Unsupported quantization type: " + quantType); } }
/** * Reads encoded probability from provided trie bit array and decodes it into actual value * for specific ngram * @param bitArr - trie bit array * @param memPtr - memory pointer for specific ngram order * @param bitOffset - offset from memPtr that is calculated according to ngram index * @param orderMinusTwo - order of ngram minus two * @return probability of ngram */ public float readProb(NgramTrieBitarr bitArr, int memPtr, int bitOffset, int orderMinusTwo) { switch (quantType) { case NO_QUANT: return bitArr.readNegativeFloat(memPtr, bitOffset); case QUANT_16: int tableIdx = orderMinusTwo * 2; if (tableIdx < tables.length - 1) bitOffset += backoffBits; return binsDecode(tableIdx, bitArr.readInt(memPtr, bitOffset, backoffMask)); //TODO implement different quantization stages default: throw new Error("Unsupported quantization type: " + quantType); } }
int readNgramWord(int ngramIdx) { int offset = ngramIdx * totalBits; return bitArr.readInt(memPtr, offset, wordMask); }
memLen += tmpLen; bitArr = new NgramTrieBitarr(memLen); this.quantProbLen = quantProbLen; this.quantProbBoLen = quantProbBoLen;
memLen += tmpLen; bitArr = new NgramTrieBitarr(memLen); this.quantProbLen = quantProbLen; this.quantProbBoLen = quantProbBoLen;
/** * Reads encoded probability from provided trie bit array and decodes it into actual value * for specific ngram * @param bitArr - trie bit array * @param memPtr - memory pointer for specific ngram order * @param bitOffset - offset from memPtr that is calculated according to ngram index * @param orderMinusTwo - order of ngram minus two * @return probability of ngram */ public float readProb(NgramTrieBitarr bitArr, int memPtr, int bitOffset, int orderMinusTwo) { switch (quantType) { case NO_QUANT: return bitArr.readNegativeFloat(memPtr, bitOffset); case QUANT_16: int tableIdx = orderMinusTwo * 2; if (tableIdx < tables.length - 1) bitOffset += backoffBits; return binsDecode(tableIdx, bitArr.readInt(memPtr, bitOffset, backoffMask)); //TODO implement different quantization stages default: throw new Error("Unsupported quantization type: " + quantType); } }
/** * Reads encoded backoff from provided trie bit array and decodes it into actual value * for specific ngram * @param bitArr - trie bit array * @param memPtr - memory pointer for specific ngram order * @param bitOffset - offset from memPtr that is calculated according to ngram index * @param orderMinusTwo - order of ngram minus two * @return backoffs of ngram */ public float readBackoff(NgramTrieBitarr bitArr, int memPtr, int bitOffset, int orderMinusTwo) { switch (quantType) { case NO_QUANT: bitOffset += 31; return bitArr.readFloat(memPtr, bitOffset); case QUANT_16: int tableIdx = orderMinusTwo * 2 + 1; return binsDecode(tableIdx, bitArr.readInt(memPtr, bitOffset, probMask)); //TODO implement different quantization stages default: throw new Error("Unsupported quantization type: " + quantType); } }
int readNgramWord(int ngramIdx) { int offset = ngramIdx * totalBits; return bitArr.readInt(memPtr, offset, wordMask); }
/** * Getter for allocated byte array to which trie is mapped * @return byte[] with ngram trie */ public byte[] getMem() { return bitArr.getArr(); }