private static final List<DataBlockEncoding> createEncodingsToIterate() { List<DataBlockEncoding> encodings = new ArrayList<>(Arrays.asList(DataBlockEncoding.values())); encodings.add(DataBlockEncoding.NONE); return Collections.unmodifiableList(encodings); }
@Parameters public static Collection<Object[]> parameters() { List<Object[]> paramList = new ArrayList<>(); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { paramList.add(new Object[] { encoding }); } return paramList; } static boolean switchKVs = false;
@Parameters public static Collection<Object[]> parameters() { List<Object[]> parameters = new ArrayList<>(); for (DataBlockEncoding dataBlockEncoding : DataBlockEncoding.values() ) { parameters.add(new Object[]{dataBlockEncoding}); } return parameters; }
@Parameters public static Collection<Object[]> parameters() { List<Object[]> paramList = new ArrayList<>(); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { for (boolean includeTags : new boolean[] { false, true }) { for (boolean compressTags : new boolean[] { false, true }) { paramList.add(new Object[] { encoding, includeTags, compressTags }); } } } return paramList; }
/** * @return All possible data block encoding configurations */ @Parameters public static Collection<Object[]> getAllConfigurations() { List<Object[]> configurations = new ArrayList<>(); for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) { for (boolean includesMemstoreTS : new boolean[] { false, true }) { HFileDataBlockEncoder dbe = (diskAlgo == DataBlockEncoding.NONE) ? NoOpDataBlockEncoder.INSTANCE : new HFileDataBlockEncoderImpl(diskAlgo); configurations.add(new Object[] { dbe, new Boolean(includesMemstoreTS) }); } } return configurations; } }
@Test public void testCrazyRandomChanges() throws Exception { prepareTest("RandomChanges"); Random rand = new Random(2934298742974297L); for (int i = 0; i < 10; ++i) { int encodingOrdinal = rand.nextInt(DataBlockEncoding.values().length); DataBlockEncoding encoding = DataBlockEncoding.values()[encodingOrdinal]; setEncodingConf(encoding, rand.nextBoolean()); writeSomeNewData(); verifyAllData(); } } }
@Test public void testDataBlockEncoding() throws Exception { for (DataBlockEncoding encoding : DataBlockEncoding.values()) { testDataBlockEncoding(encoding); } }
/** * Command line interface: * @param args Takes one argument - file size. * @throws IOException if there is a bug while reading from disk */ public static void main(final String[] args) throws IOException { if (args.length < 1) { printUsage(); System.exit(-1); } Path path = new Path(args[0]); // TODO, this test doesn't work as expected any more. Need to fix. EncodedSeekPerformanceTest utility = new EncodedSeekPerformanceTest(); utility.runTests(path, DataBlockEncoding.values()); System.exit(0); }
"'valueSize'; set on read for stats on size: Default: Not set."); System.err.println(" blockEncoding Block encoding to use. Value should be one of " + Arrays.toString(DataBlockEncoding.values()) + ". Default: NONE"); System.err.println(); System.err.println("Table Creation / Write Tests:");
/** * Create a set of column descriptors with the combination of compression, * encoding, bloom codecs available. * @param prefix family names prefix * @return the list of column descriptors */ public static List<HColumnDescriptor> generateColumnDescriptors(final String prefix) { List<HColumnDescriptor> htds = new ArrayList<>(); long familyId = 0; for (Compression.Algorithm compressionType: getSupportedCompressionAlgorithms()) { for (DataBlockEncoding encodingType: DataBlockEncoding.values()) { for (BloomType bloomType: BloomType.values()) { String name = String.format("%s-cf-!@#&-%d!@#", prefix, familyId); HColumnDescriptor htd = new HColumnDescriptor(name); htd.setCompressionType(compressionType); htd.setDataBlockEncoding(encodingType); htd.setBloomFilterType(bloomType); htds.add(htd); familyId++; } } } return htds; }
for (DataBlockEncoding encoding : DataBlockEncoding.values()) { LOG.info("Encoding: " + encoding); DataBlockEncoder encoder = encoding.getEncoder();
/** * Test whether the decompression of first key is implemented correctly. * @throws IOException */ @Test public void testFirstKeyInBlockOnSample() throws IOException { List<KeyValue> sampleKv = generator.generateTestKeyValues(NUMBER_OF_KV, includesTags); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { if (encoding.getEncoder() == null) { continue; } DataBlockEncoder encoder = encoding.getEncoder(); ByteBuffer encodedBuffer = encodeKeyValues(encoding, sampleKv, getEncodingContext(Compression.Algorithm.NONE, encoding), this.useOffheapData); Cell key = encoder.getFirstKeyCellInBlock(new SingleByteBuff(encodedBuffer)); KeyValue firstKv = sampleKv.get(0); if (0 != PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, key, firstKv)) { int commonPrefix = PrivateCellUtil.findCommonPrefixInFlatKey(key, firstKv, false, true); fail(String.format("Bug in '%s' commonPrefix %d", encoder.toString(), commonPrefix)); } } }
"regionname"), "familyname"); CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration()); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { HFileContextBuilder hcBuilder = new HFileContextBuilder(); hcBuilder.withBlockSize(2 * 1024);
byte[] rowe = Bytes.toBytes("rowe"); Table table = null; for (DataBlockEncoding encoding : DataBlockEncoding.values()) { HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor colDesc = new HColumnDescriptor(fam);
private void seekToTheKey(KeyValue expected, List<KeyValue> kvs, Cell toSeek) throws IOException { // create all seekers List<DataBlockEncoder.EncodedSeeker> encodedSeekers = new ArrayList<>(); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { if (encoding.getEncoder() == null) { continue; } DataBlockEncoder encoder = encoding.getEncoder(); HFileContext meta = new HFileContextBuilder().withHBaseCheckSum(false) .withIncludesMvcc(false).withIncludesTags(false) .withCompression(Compression.Algorithm.NONE).build(); HFileBlockEncodingContext encodingContext = encoder.newDataBlockEncodingContext(encoding, HFILEBLOCK_DUMMY_HEADER, meta); ByteBuffer encodedBuffer = TestDataBlockEncoders.encodeKeyValues(encoding, kvs, encodingContext, this.useOffheapData); DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer)); encodedSeekers.add(seeker); } // test it! // try a few random seeks checkSeekingConsistency(encodedSeekers, toSeek, expected); }
byte[] currentKey; DataBlockEncoding[] encodings = DataBlockEncoding.values();
for (DataBlockEncoding encoding: DataBlockEncoding.values()) { for (Compression.Algorithm compression: TestHFileBlock.COMPRESSION_ALGORITHMS) { HFileContext fileContext = new HFileContextBuilder()
@Test public void testGetDataBlockEncoder() throws Exception { for (DataBlockEncoding algo : DataBlockEncoding.values()) { DataBlockEncoder encoder = DataBlockEncoding.getDataBlockEncoderById(algo.getId()); if (algo.getId() != 0) { assertTrue(DataBlockEncoding.isCorrectEncoder(encoder, algo.getId())); } } try { DataBlockEncoding.getDataBlockEncoderById((short) -1); fail("Illegal encoderId, should get IllegalArgumentException."); } catch (IllegalArgumentException ie) { } try { DataBlockEncoding.getDataBlockEncoderById(Byte.MAX_VALUE); // fail because idArray[Byte.MAX_VALUE] = null fail("Illegal encoderId, should get IllegalArgumentException."); } catch (IllegalArgumentException ie) { } }
private void testEncodersOnDataset(List<KeyValue> kvList, boolean includesMemstoreTS, boolean includesTags) throws IOException { ByteBuffer unencodedDataBuf = RedundantKVGenerator.convertKvToByteBuffer(kvList, includesMemstoreTS); HFileContext fileContext = new HFileContextBuilder().withIncludesMvcc(includesMemstoreTS) .withIncludesTags(includesTags).build(); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { DataBlockEncoder encoder = encoding.getEncoder(); if (encoder == null) { continue; } HFileBlockEncodingContext encodingContext = new HFileBlockDefaultEncodingContext(encoding, HFILEBLOCK_DUMMY_HEADER, fileContext); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(HFILEBLOCK_DUMMY_HEADER); DataOutputStream dos = new DataOutputStream(baos); encoder.startBlockEncoding(encodingContext, dos); for (KeyValue kv : kvList) { encoder.encode(kv, encodingContext, dos); } encoder.endBlockEncoding(encodingContext, dos, baos.getBuffer()); byte[] encodedData = baos.toByteArray(); testAlgorithm(encodedData, unencodedDataBuf, encoder); } }
List<KeyValue> sampleKv = generator.generateTestKeyValues(NUMBER_OF_KV, includesTags); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { if (encoding.getEncoder() == null) { continue;