/** * Returns the number of bytes needed to store the checksums for * a specified data size * @param datasize number of bytes of data * @param bytesPerChecksum number of bytes in a checksum chunk * @return The number of bytes needed to store the checksum values */ static long numBytes(long datasize, int bytesPerChecksum) { return numChunks(datasize, bytesPerChecksum) * HFileBlock.CHECKSUM_SIZE; }
/** * Calculate the number of bytes required to store all the checksums * for this block. Each checksum value is a 4 byte integer. */ int totalChecksumBytes() { // If the hfile block has minorVersion 0, then there are no checksum // data to validate. Similarly, a zero value in this.bytesPerChecksum // indicates that cached blocks do not have checksum data because // checksums were already validated when the block was read from disk. if (!fileContext.isUseHBaseChecksum() || this.fileContext.getBytesPerChecksum() == 0) { return 0; } return (int) ChecksumUtil.numBytes(onDiskDataSizeWithHeader, this.fileContext.getBytesPerChecksum()); }
ChecksumUtil.generateExceptionForChecksumFailureForTest(true);
compressAndEncryptDat.getOffset(), compressAndEncryptDat.getLength()); int numBytes = (int) ChecksumUtil.numBytes( onDiskBlockBytesWithHeader.size(), fileContext.getBytesPerChecksum()); onDiskChecksum = new byte[numBytes]; ChecksumUtil.generateChecksums( onDiskBlockBytesWithHeader.getBuffer(), 0,onDiskBlockBytesWithHeader.size(), onDiskChecksum, 0, fileContext.getChecksumType(), fileContext.getBytesPerChecksum());
ChecksumUtil.reserveSpaceForChecksums(compressedByteStream, onDiskDataSizeWithHeader, bytesPerChecksum); ChecksumUtil.generateChecksums( onDiskBytesWithHeader, 0, onDiskDataSizeWithHeader, onDiskBytesWithHeader, onDiskDataSizeWithHeader, int numBytes = (int)ChecksumUtil.numBytes( uncompressedBytesWithHeader.length, bytesPerChecksum); uncompressedBytesWithHeader.length, onDiskDataSizeWithHeader); ChecksumUtil.generateChecksums( uncompressedBytesWithHeader, 0, uncompressedBytesWithHeader.length, onDiskChecksum, 0,
/** * Generates the checksum for the header as well as the data and * then validates that it matches the value stored in the header. * If there is a checksum mismatch, then return false. Otherwise * return true. */ protected boolean validateBlockChecksum(HFileBlock block, long offset, byte[] data, int hdrSize) throws IOException { return ChecksumUtil.validateBlockChecksum(path, offset, block, data, hdrSize); }
/** * Generates the checksum for the header as well as the data and then validates it. * If the block doesn't uses checksum, returns false. * @return True if checksum matches, else false. */ private boolean validateChecksum(long offset, ByteBuffer data, int hdrSize) throws IOException { // If this is an older version of the block that does not have checksums, then return false // indicating that checksum verification did not succeed. Actually, this method should never // be called when the minorVersion is 0, thus this is a defensive check for a cannot-happen // case. Since this is a cannot-happen case, it is better to return false to indicate a // checksum validation failure. if (!fileContext.isUseHBaseChecksum()) { return false; } return ChecksumUtil.validateChecksum(data, pathName, offset, hdrSize); }
.compressAndEncrypt(uncompressedBytesWithHeader); int numBytes = (int) ChecksumUtil.numBytes( onDiskBytesWithHeader.length, fileContext.getBytesPerChecksum()); ChecksumUtil.generateChecksums( onDiskBytesWithHeader, 0, onDiskBytesWithHeader.length, onDiskChecksum, 0, fileContext.getChecksumType(), fileContext.getBytesPerChecksum());
/** * Generates the checksum for the header as well as the data and * then validates that it matches the value stored in the header. * If there is a checksum mismatch, then return false. Otherwise * return true. */ protected boolean validateBlockChecksum(HFileBlock block, byte[] data, int hdrSize) throws IOException { return ChecksumUtil.validateBlockChecksum(path, block, data, hdrSize); } }
/** * Write dummy checksums to the end of the specified bytes array * to reserve space for writing checksums later * @param baos OutputStream to write dummy checkum values * @param numBytes Number of bytes of data for which dummy checksums * need to be generated * @param bytesPerChecksum Number of bytes per checksum value */ static void reserveSpaceForChecksums(ByteArrayOutputStream baos, int numBytes, int bytesPerChecksum) throws IOException { long numChunks = numChunks(numBytes, bytesPerChecksum); long bytesLeft = numChunks * HFileBlock.CHECKSUM_SIZE; while (bytesLeft > 0) { long count = Math.min(bytesLeft, DUMMY_VALUE.length); baos.write(DUMMY_VALUE, 0, (int)count); bytesLeft -= count; } }
/** * Clones the header followed by the uncompressed data, even if using * compression. This is needed for storing uncompressed blocks in the block * cache. Can be called in the "writing" state or the "block ready" state. * Returns only the header and data, does not include checksum data. * * @return Returns a copy of uncompressed block bytes for caching on write */ @VisibleForTesting ByteBuffer cloneUncompressedBufferWithHeader() { expectState(State.BLOCK_READY); byte[] uncompressedBlockBytesWithHeader = baosInMemory.toByteArray(); int numBytes = (int) ChecksumUtil.numBytes( onDiskBlockBytesWithHeader.size(), fileContext.getBytesPerChecksum()); putHeader(uncompressedBlockBytesWithHeader, 0, onDiskBlockBytesWithHeader.size() + numBytes, baosInMemory.size(), onDiskBlockBytesWithHeader.size()); return ByteBuffer.wrap(uncompressedBlockBytesWithHeader); }
public HBaseTestingUtility(Configuration conf) { this.conf = conf; // a hbase checksum verification failure will cause unit tests to fail ChecksumUtil.generateExceptionForChecksumFailureForTest(true); setHDFSClientRetryProperty(); }
os.close(); long expectedChunks = ChecksumUtil.numChunks( dataSize + HConstants.HFILEBLOCK_HEADER_SIZE, bytesPerChecksum);
/** * Calcuate the number of bytes required to store all the checksums * for this block. Each checksum value is a 4 byte integer. */ int totalChecksumBytes() { // If the hfile block has minorVersion 0, then there are no checksum // data to validate. Similarly, a zero value in this.bytesPerChecksum // indicates that cached blocks do not have checksum data because // checksums were already validated when the block was read from disk. if (minorVersion < MINOR_VERSION_WITH_CHECKSUM || this.bytesPerChecksum == 0) { return 0; } return (int)ChecksumUtil.numBytes(onDiskDataSizeWithHeader, bytesPerChecksum); }
ChecksumUtil.generateExceptionForChecksumFailureForTest(true);
/** * Returns the number of bytes needed to store the checksums for * a specified data size * @param datasize number of bytes of data * @param bytesPerChecksum number of bytes in a checksum chunk * @return The number of bytes needed to store the checksum values */ static long numBytes(long datasize, int bytesPerChecksum) { return numChunks(datasize, bytesPerChecksum) * HFileBlock.CHECKSUM_SIZE; }
/** * Calcuate the number of bytes required to store all the checksums * for this block. Each checksum value is a 4 byte integer. */ int totalChecksumBytes() { // If the hfile block has minorVersion 0, then there are no checksum // data to validate. Similarly, a zero value in this.bytesPerChecksum // indicates that cached blocks do not have checksum data because // checksums were already validated when the block was read from disk. if (!fileContext.isUseHBaseChecksum() || this.fileContext.getBytesPerChecksum() == 0) { return 0; } return (int) ChecksumUtil.numBytes(onDiskDataSizeWithHeader, this.fileContext.getBytesPerChecksum()); }
/** * Returns the number of bytes needed to store the checksums for * a specified data size * @param datasize number of bytes of data * @param bytesPerChecksum number of bytes in a checksum chunk * @return The number of bytes needed to store the checksum values */ static long numBytes(long datasize, int bytesPerChecksum) { return numChunks(datasize, bytesPerChecksum) * HFileBlock.CHECKSUM_SIZE; }
/** * Write dummy checksums to the end of the specified bytes array * to reserve space for writing checksums later * @param baos OutputStream to write dummy checkum values * @param numBytes Number of bytes of data for which dummy checksums * need to be generated * @param bytesPerChecksum Number of bytes per checksum value */ static void reserveSpaceForChecksums(ByteArrayOutputStream baos, int numBytes, int bytesPerChecksum) throws IOException { long numChunks = numChunks(numBytes, bytesPerChecksum); long bytesLeft = numChunks * HFileBlock.CHECKSUM_SIZE; while (bytesLeft > 0) { long count = Math.min(bytesLeft, DUMMY_VALUE.length); baos.write(DUMMY_VALUE, 0, (int)count); bytesLeft -= count; } }
/** * Write dummy checksums to the end of the specified bytes array * to reserve space for writing checksums later * @param baos OutputStream to write dummy checkum values * @param numBytes Number of bytes of data for which dummy checksums * need to be generated * @param bytesPerChecksum Number of bytes per checksum value */ static void reserveSpaceForChecksums(ByteArrayOutputStream baos, int numBytes, int bytesPerChecksum) throws IOException { long numChunks = numChunks(numBytes, bytesPerChecksum); long bytesLeft = numChunks * HFileBlock.CHECKSUM_SIZE; while (bytesLeft > 0) { long count = Math.min(bytesLeft, DUMMY_VALUE.length); baos.write(DUMMY_VALUE, 0, (int)count); bytesLeft -= count; } }