public static void doSmokeTest(FileSystem fs, Path path, String codec) throws Exception { Configuration conf = HBaseConfiguration.create(); HFileContext context = new HFileContextBuilder() .withCompression(HFileWriterImpl.compressionByName(codec)).build(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, path) .withFileContext(context) .create(); // Write any-old Cell... final byte [] rowKey = Bytes.toBytes("compressiontestkey"); Cell c = CellUtil.createCell(rowKey, Bytes.toBytes("compressiontestval")); writer.append(c); writer.appendFileInfo(Bytes.toBytes("compressioninfokey"), Bytes.toBytes("compressioninfoval")); writer.close(); Cell cc = null; HFile.Reader reader = HFile.createReader(fs, path, CacheConfig.DISABLED, true, conf); try { reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); scanner.seekTo(); // position to the start of file // Scanner does not do Cells yet. Do below for now till fixed. cc = scanner.getCell(); if (CellComparator.getInstance().compareRows(c, cc) != 0) { throw new Exception("Read back incorrect result: " + c.toString() + " vs " + cc.toString()); } } finally { reader.close(); } }
fs.mkdirs(hfilePath); Path path = new Path(pathStr); HFile.WriterFactory wf = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration()); Assert.assertNotNull(wf); HFileContext context = new HFileContext();
private String createHFileForFamilies(Path testPath, byte[] family, byte[] valueBytes) throws IOException { HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration()); // TODO We need a way to do this without creating files Path testFile = new Path(testPath, TEST_UTIL.getRandomUUID().toString()); FSDataOutputStream out = TEST_UTIL.getTestFileSystem().create(testFile); try { hFileFactory.withOutputStream(out); hFileFactory.withFileContext(new HFileContext()); HFile.Writer writer = hFileFactory.create(); try { writer.append(new KeyValue(CellUtil.createCell(valueBytes, family, valueBytes, 0L, KeyValue.Type.Put.getCode(), valueBytes))); } finally { writer.close(); } } finally { out.close(); } return testFile.toString(); }
private String createHFileForFamilies(byte[] family) throws IOException { HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(conf); // TODO We need a way to do this without creating files File hFileLocation = testFolder.newFile(); FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(hFileLocation), null); try { hFileFactory.withOutputStream(out); hFileFactory.withFileContext(new HFileContext()); HFile.Writer writer = hFileFactory.create(); try { writer.append(new KeyValue(CellUtil.createCell(randomBytes, family, randomBytes, 0L, KeyValue.Type.Put.getCode(), randomBytes))); } finally { writer.close(); } } finally { out.close(); } return hFileLocation.getAbsoluteFile().getAbsolutePath(); }
@Override void setUp() throws Exception { HFileContextBuilder builder = new HFileContextBuilder() .withCompression(HFileWriterImpl.compressionByName(codec)) .withBlockSize(RFILE_BLOCKSIZE); if (cipher == "aes") { byte[] cipherKey = new byte[AES.KEY_LENGTH]; new SecureRandom().nextBytes(cipherKey); builder.withEncryptionContext(Encryption.newContext(conf) .setCipher(Encryption.getCipher(conf, cipher)) .setKey(cipherKey)); } else if (!"none".equals(cipher)) { throw new IOException("Cipher " + cipher + " not supported."); } HFileContext hFileContext = builder.build(); writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, mf) .withFileContext(hFileContext) .withComparator(CellComparator.getInstance()) .create(); }
.withCompression(HFileWriterImpl.compressionByName(options.compress)) .build(); Writer writer = HFile.getWriterFactoryNoCache(conf) .withOutputStream(fout) .withFileContext(context)
Path makeNewFile(TagUsage tagUsage) throws IOException { Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile"); FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile); int blocksize = toKV("a", tagUsage).getLength() * 3; HFileContext context = new HFileContextBuilder().withBlockSize(blocksize) .withDataBlockEncoding(encoding) .withIncludesTags(true).build(); Configuration conf = TEST_UTIL.getConfiguration(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout) .withFileContext(context) .withComparator(CellComparatorImpl.COMPARATOR).create(); // 4 bytes * 3 * 2 for each key/value + // 3 for keys, 15 for values = 42 (woot) writer.append(toKV("c", tagUsage)); writer.append(toKV("e", tagUsage)); writer.append(toKV("g", tagUsage)); // block transition writer.append(toKV("i", tagUsage)); writer.append(toKV("k", tagUsage)); writer.close(); fout.close(); return ncTFile; }
public static void doSmokeTest(FileSystem fs, Path path, String codec) throws Exception { Configuration conf = HBaseConfiguration.create(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, path) .withCompression(codec) .create(); writer.append(Bytes.toBytes("testkey"), Bytes.toBytes("testval")); writer.appendFileInfo(Bytes.toBytes("infokey"), Bytes.toBytes("infoval")); writer.close(); HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf)); reader.loadFileInfo(); byte[] key = reader.getFirstKey(); boolean rc = Bytes.toString(key).equals("testkey"); reader.close(); if (!rc) { throw new Exception("Read back incorrect result: " + Bytes.toStringBinary(key)); } }
public static void doSmokeTest(FileSystem fs, Path path, String codec) throws Exception { Configuration conf = HBaseConfiguration.create(); HFileContext context = new HFileContextBuilder() .withCompression(AbstractHFileWriter.compressionByName(codec)).build(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, path) .withFileContext(context) .create(); // Write any-old Cell... final byte [] rowKey = Bytes.toBytes("compressiontestkey"); Cell c = CellUtil.createCell(rowKey, Bytes.toBytes("compressiontestval")); writer.append(c); writer.appendFileInfo(Bytes.toBytes("compressioninfokey"), Bytes.toBytes("compressioninfoval")); writer.close(); Cell cc = null; HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf); try { reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); scanner.seekTo(); // position to the start of file // Scanner does not do Cells yet. Do below for now till fixed. cc = scanner.getKeyValue(); if (CellComparator.compareRows(c, cc) != 0) { throw new Exception("Read back incorrect result: " + c.toString() + " vs " + cc.toString()); } } finally { reader.close(); } }
fs.mkdirs(hfilePath); Path path = new Path(pathStr); HFile.WriterFactory wf = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration()); Assert.assertNotNull(wf); HFileContext context = new HFileContext();
private String createHFileForFamilies(byte[] family) throws IOException { HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(conf); // TODO We need a way to do this without creating files File hFileLocation = testFolder.newFile(); FSDataOutputStream out = new FSDataOutputStream(new FileOutputStream(hFileLocation), null); try { hFileFactory.withOutputStream(out); hFileFactory.withFileContext(new HFileContext()); HFile.Writer writer = hFileFactory.create(); try { writer.append(new KeyValue(CellUtil.createCell(randomBytes, family, randomBytes, 0L, KeyValue.Type.Put.getCode(), randomBytes))); } finally { writer.close(); } } finally { out.close(); } return hFileLocation.getAbsoluteFile().getAbsolutePath(); }
private String createHFileForFamilies(Path testPath, byte[] family, byte[] valueBytes) throws IOException { HFile.WriterFactory hFileFactory = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration()); // TODO We need a way to do this without creating files Path testFile = new Path(testPath, TEST_UTIL.getRandomUUID().toString()); FSDataOutputStream out = TEST_UTIL.getTestFileSystem().create(testFile); try { hFileFactory.withOutputStream(out); hFileFactory.withFileContext(new HFileContext()); HFile.Writer writer = hFileFactory.create(); try { writer.append(new KeyValue(CellUtil.createCell(valueBytes, family, valueBytes, 0L, KeyValue.Type.Put.getCode(), valueBytes))); } finally { writer.close(); } } finally { out.close(); } return testFile.toString(); }
@Override void setUp() throws Exception { HFileContextBuilder builder = new HFileContextBuilder() .withCompression(HFileWriterImpl.compressionByName(codec)) .withBlockSize(RFILE_BLOCKSIZE); if (cipher == "aes") { byte[] cipherKey = new byte[AES.KEY_LENGTH]; new SecureRandom().nextBytes(cipherKey); builder.withEncryptionContext(Encryption.newContext(conf) .setCipher(Encryption.getCipher(conf, cipher)) .setKey(cipherKey)); } else if (!"none".equals(cipher)) { throw new IOException("Cipher " + cipher + " not supported."); } HFileContext hFileContext = builder.build(); writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, mf) .withFileContext(hFileContext) .withComparator(CellComparator.getInstance()) .create(); }
.withCompression(HFileWriterImpl.compressionByName(options.compress)) .build(); Writer writer = HFile.getWriterFactoryNoCache(conf) .withOutputStream(fout) .withFileContext(context)
Path makeNewFile(TagUsage tagUsage) throws IOException { Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile"); FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile); int blocksize = toKV("a", tagUsage).getLength() * 3; HFileContext context = new HFileContextBuilder().withBlockSize(blocksize) .withDataBlockEncoding(encoding) .withIncludesTags(true).build(); Configuration conf = TEST_UTIL.getConfiguration(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout) .withFileContext(context) .withComparator(CellComparatorImpl.COMPARATOR).create(); // 4 bytes * 3 * 2 for each key/value + // 3 for keys, 15 for values = 42 (woot) writer.append(toKV("c", tagUsage)); writer.append(toKV("e", tagUsage)); writer.append(toKV("g", tagUsage)); // block transition writer.append(toKV("i", tagUsage)); writer.append(toKV("k", tagUsage)); writer.close(); fout.close(); return ncTFile; }