.withComparator(comparator) .withFavoredNodes(favoredNodes) .withFileContext(fileContext)
.withFileContext(context) .withComparator(CellComparatorImpl.COMPARATOR) .create(); int numberOfKeys = 1000;
.withOutputStream(fout) .withFileContext(meta) .withComparator(CellComparatorImpl.COMPARATOR) .create(); LOG.info(Objects.toString(writer));
.withPath(fs, hfilePath) .withFileContext(context) .withComparator(CellComparatorImpl.COMPARATOR) .create();
.withFileContext(context) .withComparator(CellComparatorImpl.COMPARATOR) .create(); int numberOfKeys = 1000;
.withCompression(compress) .withDataBlockEncoder(dataBlockEncoder) .withComparator(comparator.getRawComparator()) .withChecksumType(checksumType) .withBytesPerChecksum(bytesPerChecksum)
writer = HFile.getWriterFactory(conf, cacheConf) .withPath(fs, path) .withComparator(comparator) .withFavoredNodes(favoredNodes) .withFileContext(fileContext)
.withOutputStream(fout) .withFileContext(meta) .withComparator(CellComparatorImpl.COMPARATOR) .create(); LOG.info(Objects.toString(writer));
.withOutputStream(fout) .withFileContext(context) .withComparator(CellComparatorImpl.COMPARATOR) .create(); try {
/** * Create an HFile with the given number of rows between a given * start key and end key. */ public static void createHFile( Configuration conf, FileSystem fs, Path path, byte[] family, byte[] qualifier, byte[] startKey, byte[] endKey, int numRows) throws IOException { HFile.WriterFactory wf = HFile.getWriterFactory(conf, new CacheConfig(conf)); HFileContext hFileContext = new HFileContext(); wf.withFileContext(hFileContext); wf.withComparator(KeyValue.COMPARATOR); wf.withPath(fs, path); HFile.Writer writer = wf.create(); long now = System.currentTimeMillis(); try { // subtract 2 since iterateOnSplits doesn't include boundary keys for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows - 2)) { KeyValue kv = new KeyValue(key, family, qualifier, now, key); writer.append(kv); } } finally { writer.close(); } }
public HFileSortedOplogWriter(int keys) throws IOException { try { int hfileBlockSize = Integer.getInteger( HoplogConfig.HFILE_BLOCK_SIZE_CONF, (1 << 16)); Algorithm compress = Algorithm.valueOf(System.getProperty(HoplogConfig.COMPRESSION, HoplogConfig.COMPRESSION_DEFAULT)); ByteComparator bc = new ByteComparator(); HFileContext hcontext = new HFileContextBuilder() .withBlockSize(hfileBlockSize) .withCompression(compress) .build(); writer = HFile.getWriterFactory(conf, cacheConf) .withPath(fsProvider.getFS(), path) .withFileContext(hcontext) .withComparator(bc) .create(); bfw = BloomFilterFactory.createGeneralBloomAtWrite(conf, cacheConf, BloomType.ROW, keys, writer); logger.fine("Created hoplog writer with compression " + compress); } catch (IOException e) { logger.fine("IO Error while creating writer"); throw e; } }
Path makeNewFile(TagUsage tagUsage) throws IOException { Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile"); FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile); int blocksize = toKV("a", tagUsage).getLength() * 3; HFileContext context = new HFileContextBuilder().withBlockSize(blocksize) .withDataBlockEncoding(encoding) .withIncludesTags(true).build(); Configuration conf = TEST_UTIL.getConfiguration(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout) .withFileContext(context) .withComparator(CellComparatorImpl.COMPARATOR).create(); // 4 bytes * 3 * 2 for each key/value + // 3 for keys, 15 for values = 42 (woot) writer.append(toKV("c", tagUsage)); writer.append(toKV("e", tagUsage)); writer.append(toKV("g", tagUsage)); // block transition writer.append(toKV("i", tagUsage)); writer.append(toKV("k", tagUsage)); writer.close(); fout.close(); return ncTFile; }
public HFileSortedOplogWriter() throws IOException { HFileContext hcontext = new HFileContextBuilder() .withBlockSize(sopConfig.getBlockSize()) .withBytesPerCheckSum(sopConfig.getBytesPerChecksum()) .withChecksumType(HFileSortedOplogFactory.convertChecksum( sopConfig.getChecksum())) .withCompression(HFileSortedOplogFactory.convertCompression( sopConfig.getCompression())) .withDataBlockEncoding(HFileSortedOplogFactory.convertEncoding( sopConfig.getKeyEncoding()).getDataBlockEncoding()) .build(); writer = HFile.getWriterFactory(hconf, hcache) .withPath(fs, path) .withFileContext(hcontext) .withComparator(sopConfig.getComparator()) .create(); bfw = sopConfig.isBloomFilterEnabled() ? // BloomFilterFactory.createGeneralBloomAtWrite(hconf, hcache, BloomType.ROW, // 0, writer, sopConfig.getComparator()) BloomFilterFactory.createGeneralBloomAtWrite(hconf, hcache, BloomType.ROW, 0, writer) : null; }
@Override void setUp() throws Exception { HFileContextBuilder builder = new HFileContextBuilder() .withCompression(HFileWriterImpl.compressionByName(codec)) .withBlockSize(RFILE_BLOCKSIZE); if (cipher == "aes") { byte[] cipherKey = new byte[AES.KEY_LENGTH]; new SecureRandom().nextBytes(cipherKey); builder.withEncryptionContext(Encryption.newContext(conf) .setCipher(Encryption.getCipher(conf, cipher)) .setKey(cipherKey)); } else if (!"none".equals(cipher)) { throw new IOException("Cipher " + cipher + " not supported."); } HFileContext hFileContext = builder.build(); writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, mf) .withFileContext(hFileContext) .withComparator(CellComparator.getInstance()) .create(); }
.withPath(fs, hfilePath) .withFileContext(context) .withComparator(CellComparatorImpl.COMPARATOR) .create();
.withOutputStream(fout) .withFileContext(context) .withComparator(CellComparatorImpl.COMPARATOR) .create(); try {
Path makeNewFile(TagUsage tagUsage) throws IOException { Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile"); FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile); int blocksize = toKV("a", tagUsage).getLength() * 3; HFileContext context = new HFileContextBuilder().withBlockSize(blocksize) .withDataBlockEncoding(encoding) .withIncludesTags(true).build(); Configuration conf = TEST_UTIL.getConfiguration(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout) .withFileContext(context) .withComparator(CellComparatorImpl.COMPARATOR).create(); // 4 bytes * 3 * 2 for each key/value + // 3 for keys, 15 for values = 42 (woot) writer.append(toKV("c", tagUsage)); writer.append(toKV("e", tagUsage)); writer.append(toKV("g", tagUsage)); // block transition writer.append(toKV("i", tagUsage)); writer.append(toKV("k", tagUsage)); writer.close(); fout.close(); return ncTFile; }
@Override void setUp() throws Exception { HFileContextBuilder builder = new HFileContextBuilder() .withCompression(HFileWriterImpl.compressionByName(codec)) .withBlockSize(RFILE_BLOCKSIZE); if (cipher == "aes") { byte[] cipherKey = new byte[AES.KEY_LENGTH]; new SecureRandom().nextBytes(cipherKey); builder.withEncryptionContext(Encryption.newContext(conf) .setCipher(Encryption.getCipher(conf, cipher)) .setKey(cipherKey)); } else if (!"none".equals(cipher)) { throw new IOException("Cipher " + cipher + " not supported."); } HFileContext hFileContext = builder.build(); writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, mf) .withFileContext(hFileContext) .withComparator(CellComparator.getInstance()) .create(); }