@Test public void testCommonPrefixComparators() { KeyValue kv1 = new KeyValue(row1, fam1, qual1, 1L, Type.Put); KeyValue kv2 = new KeyValue(row1, fam_1_2, qual1, 1L, Type.Maximum); assertTrue((BufferedDataBlockEncoder.compareCommonFamilyPrefix(kv1, kv2, 4) < 0)); kv1 = new KeyValue(row1, fam1, qual1, 1L, Type.Put); kv2 = new KeyValue(row_1_0, fam_1_2, qual1, 1L, Type.Maximum); assertTrue((BufferedDataBlockEncoder.compareCommonRowPrefix(kv1, kv2, 4) < 0)); kv1 = new KeyValue(row1, fam1, qual2, 1L, Type.Put); kv2 = new KeyValue(row1, fam1, qual1, 1L, Type.Maximum); assertTrue((BufferedDataBlockEncoder.compareCommonQualifierPrefix(kv1, kv2, 4) > 0)); }
@Override public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext blkDecodingCtx) throws IOException { if (blkDecodingCtx.getClass() != HFileBlockDefaultDecodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultDecodingContext.class.getName() + " as the decoding context."); } HFileBlockDefaultDecodingContext decodingCtx = (HFileBlockDefaultDecodingContext) blkDecodingCtx; if (decodingCtx.getHFileContext().isIncludesTags() && decodingCtx.getHFileContext().isCompressTags()) { if (decodingCtx.getTagCompressionContext() != null) { // It will be overhead to create the TagCompressionContext again and again for every block // decoding. decodingCtx.getTagCompressionContext().clear(); } else { try { TagCompressionContext tagCompressionContext = new TagCompressionContext( LRUDictionary.class, Byte.MAX_VALUE); decodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); } } } return internalDecodeKeyValues(source, 0, 0, decodingCtx); }
@Override public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException { BufferedDataBlockEncodingState state = (BufferedDataBlockEncodingState) encodingCtx .getEncodingState(); int encodedKvSize = internalEncode(cell, (HFileBlockDefaultEncodingContext) encodingCtx, out); state.unencodedDataSizeWritten += encodedKvSize; return encodedKvSize; }
@Override public void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out, byte[] uncompressedBytesWithHeader) throws IOException { BufferedDataBlockEncodingState state = (BufferedDataBlockEncodingState) encodingCtx .getEncodingState(); // Write the unencodedDataSizeWritten (with header size) Bytes.putInt(uncompressedBytesWithHeader, HConstants.HFILEBLOCK_HEADER_SIZE + DataBlockEncoding.ID_SIZE, state.unencodedDataSizeWritten ); postEncoding(encodingCtx); }
@Override public ByteBuffer uncompressKeyValues(DataInputStream source, boolean includesMemstoreTS) throws IOException { return uncompressKeyValues(source, 0, 0, includesMemstoreTS); }
@Override public void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out, byte[] uncompressedBytesWithHeader) throws IOException { BufferedDataBlockEncodingState state = (BufferedDataBlockEncodingState) encodingCtx .getEncodingState(); // Write the unencodedDataSizeWritten (with header size) Bytes.putInt(uncompressedBytesWithHeader, HConstants.HFILEBLOCK_HEADER_SIZE + DataBlockEncoding.ID_SIZE, state.unencodedDataSizeWritten ); postEncoding(encodingCtx); }
comp = compareCommonRowPrefix(seekCell, keyOnlyKV, rowCommonPrefix); if (comp == 0) { comp = compareTypeBytes(seekCell, keyOnlyKV); familyCommonPrefix += findCommonPrefixInFamilyPart(seekCell, keyOnlyKV, familyCommonPrefix); comp = compareCommonFamilyPrefix(seekCell, keyOnlyKV, familyCommonPrefix); if (comp == 0) { qualCommonPrefix += findCommonPrefixInQualifierPart(seekCell, keyOnlyKV, qualCommonPrefix); comp = compareCommonQualifierPrefix(seekCell, keyOnlyKV, qualCommonPrefix); if (comp == 0) { comp = CellComparator.getInstance().compareTimestamps(seekCell, keyOnlyKV);
@Override public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException { BufferedDataBlockEncodingState state = (BufferedDataBlockEncodingState) encodingCtx .getEncodingState(); int encodedKvSize = internalEncode(cell, (HFileBlockDefaultEncodingContext) encodingCtx, out); state.unencodedDataSizeWritten += encodedKvSize; return encodedKvSize; }
@Override public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext blkDecodingCtx) throws IOException { if (blkDecodingCtx.getClass() != HFileBlockDefaultDecodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultDecodingContext.class.getName() + " as the decoding context."); } HFileBlockDefaultDecodingContext decodingCtx = (HFileBlockDefaultDecodingContext) blkDecodingCtx; if (decodingCtx.getHFileContext().isIncludesTags() && decodingCtx.getHFileContext().isCompressTags()) { if (decodingCtx.getTagCompressionContext() != null) { // It will be overhead to create the TagCompressionContext again and again for every block // decoding. decodingCtx.getTagCompressionContext().clear(); } else { try { TagCompressionContext tagCompressionContext = new TagCompressionContext( LRUDictionary.class, Byte.MAX_VALUE); decodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); } } } return internalDecodeKeyValues(source, 0, 0, decodingCtx); }
@Override public void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out, byte[] uncompressedBytesWithHeader) throws IOException { BufferedDataBlockEncodingState state = (BufferedDataBlockEncodingState) encodingCtx .getEncodingState(); // Write the unencodedDataSizeWritten (with header size) Bytes.putInt(uncompressedBytesWithHeader, HConstants.HFILEBLOCK_HEADER_SIZE + DataBlockEncoding.ID_SIZE, state.unencodedDataSizeWritten ); postEncoding(encodingCtx); }
@Test public void testCommonPrefixComparators() { KeyValue kv1 = new KeyValue(row1, fam1, qual1, 1L, Type.Put); KeyValue kv2 = new KeyValue(row1, fam_1_2, qual1, 1L, Type.Maximum); assertTrue((BufferedDataBlockEncoder.compareCommonFamilyPrefix(kv1, kv2, 4) < 0)); kv1 = new KeyValue(row1, fam1, qual1, 1L, Type.Put); kv2 = new KeyValue(row_1_0, fam_1_2, qual1, 1L, Type.Maximum); assertTrue((BufferedDataBlockEncoder.compareCommonRowPrefix(kv1, kv2, 4) < 0)); kv1 = new KeyValue(row1, fam1, qual2, 1L, Type.Put); kv2 = new KeyValue(row1, fam1, qual1, 1L, Type.Maximum); assertTrue((BufferedDataBlockEncoder.compareCommonQualifierPrefix(kv1, kv2, 4) > 0)); }
@Override public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException { BufferedDataBlockEncodingState state = (BufferedDataBlockEncodingState) encodingCtx .getEncodingState(); int encodedKvSize = internalEncode(cell, (HFileBlockDefaultEncodingContext) encodingCtx, out); state.unencodedDataSizeWritten += encodedKvSize; return encodedKvSize; }
@Override public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext blkDecodingCtx) throws IOException { if (blkDecodingCtx.getClass() != HFileBlockDefaultDecodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultDecodingContext.class.getName() + " as the decoding context."); } HFileBlockDefaultDecodingContext decodingCtx = (HFileBlockDefaultDecodingContext) blkDecodingCtx; if (decodingCtx.getHFileContext().isIncludesTags() && decodingCtx.getHFileContext().isCompressTags()) { if (decodingCtx.getTagCompressionContext() != null) { // It will be overhead to create the TagCompressionContext again and again for every block // decoding. decodingCtx.getTagCompressionContext().clear(); } else { try { TagCompressionContext tagCompressionContext = new TagCompressionContext( LRUDictionary.class, Byte.MAX_VALUE); decodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); } } } return internalDecodeKeyValues(source, 0, 0, decodingCtx); }
comp = compareCommonRowPrefix(seekCell, keyOnlyKV, rowCommonPrefix); if (comp == 0) { comp = compareTypeBytes(seekCell, keyOnlyKV); familyCommonPrefix += findCommonPrefixInFamilyPart(seekCell, keyOnlyKV, familyCommonPrefix); comp = compareCommonFamilyPrefix(seekCell, keyOnlyKV, familyCommonPrefix); if (comp == 0) { qualCommonPrefix += findCommonPrefixInQualifierPart(seekCell, keyOnlyKV, qualCommonPrefix); comp = compareCommonQualifierPrefix(seekCell, keyOnlyKV, qualCommonPrefix); if (comp == 0) { comp = CellComparator.getInstance().compareTimestamps(seekCell, keyOnlyKV);
@Override public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out) throws IOException { BufferedDataBlockEncodingState state = (BufferedDataBlockEncodingState) encodingCtx .getEncodingState(); int encodedKvSize = internalEncode(cell, (HFileBlockDefaultEncodingContext) encodingCtx, out); state.unencodedDataSizeWritten += encodedKvSize; return encodedKvSize; }
@Override public ByteBuffer decodeKeyValues(DataInputStream source, HFileBlockDecodingContext blkDecodingCtx) throws IOException { if (blkDecodingCtx.getClass() != HFileBlockDefaultDecodingContext.class) { throw new IOException(this.getClass().getName() + " only accepts " + HFileBlockDefaultDecodingContext.class.getName() + " as the decoding context."); } HFileBlockDefaultDecodingContext decodingCtx = (HFileBlockDefaultDecodingContext) blkDecodingCtx; if (decodingCtx.getHFileContext().isIncludesTags() && decodingCtx.getHFileContext().isCompressTags()) { if (decodingCtx.getTagCompressionContext() != null) { // It will be overhead to create the TagCompressionContext again and again for every block // decoding. decodingCtx.getTagCompressionContext().clear(); } else { try { TagCompressionContext tagCompressionContext = new TagCompressionContext( LRUDictionary.class, Byte.MAX_VALUE); decodingCtx.setTagCompressionContext(tagCompressionContext); } catch (Exception e) { throw new IOException("Failed to initialize TagCompressionContext", e); } } } return internalDecodeKeyValues(source, 0, 0, decodingCtx); }
comp = compareCommonRowPrefix(seekCell, keyOnlyKV, rowCommonPrefix); if (comp == 0) { comp = compareTypeBytes(seekCell, keyOnlyKV); familyCommonPrefix += findCommonPrefixInFamilyPart(seekCell, keyOnlyKV, familyCommonPrefix); comp = compareCommonFamilyPrefix(seekCell, keyOnlyKV, familyCommonPrefix); if (comp == 0) { qualCommonPrefix += findCommonPrefixInQualifierPart(seekCell, keyOnlyKV, qualCommonPrefix); comp = compareCommonQualifierPrefix(seekCell, keyOnlyKV, qualCommonPrefix); if (comp == 0) { comp = CellComparator.getInstance().compareTimestamps(seekCell, keyOnlyKV);