/** * Returns the factory to be used to create {@link HFile} writers. * Disables block cache access for all writers created through the * returned factory. */ public static final WriterFactory getWriterFactoryNoCache(Configuration conf) { return HFile.getWriterFactory(conf, CacheConfig.DISABLED); }
this.timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC); writer = HFile.getWriterFactory(conf, cacheConf) .withPath(fs, path) .withComparator(comparator)
/** * Create an HFile with the given number of bytes */ private void createHFile(Path path, int rowIdx, byte[] dummyData) throws IOException { HFileContext meta = new HFileContextBuilder().build(); HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)).withPath(fs, path) .withFileContext(meta).create(); long now = System.currentTimeMillis(); try { KeyValue kv = new KeyValue(Bytes.add(STARTROW, Bytes.toBytes(rowIdx)), COLUMN_FAMILY, Bytes.toBytes("colX"), now, dummyData); writer.append(kv); } finally { writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis())); writer.close(); } }
private static void createHFile(Configuration conf, FileSystem fs, Path path, byte[] family, byte[] qualifier) throws IOException { HFileContext context = new HFileContextBuilder().build(); HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf)).withPath(fs, path) .withFileContext(context).create(); long now = System.currentTimeMillis(); try { for (int i = 1; i <= 9; i++) { KeyValue kv = new KeyValue(Bytes.toBytes(i + ""), family, qualifier, now, Bytes.toBytes(i + "")); writer.append(kv); } } finally { writer.close(); } } }
HFile.Writer writer = HFile.getWriterFactory( TEST_UTIL.getConfiguration(), cacheConf) .withOutputStream(fout)
private void metablocks(final String compress) throws Exception { Path mFile = new Path(ROOT_DIR, "meta.hfile"); FSDataOutputStream fout = createFSOutput(mFile); HFileContext meta = new HFileContextBuilder() .withCompression(HFileWriterImpl.compressionByName(compress)) .withBlockSize(minBlockSize).build(); Writer writer = HFile.getWriterFactory(conf, cacheConf) .withOutputStream(fout) .withFileContext(meta) .create(); someTestingWithMetaBlock(writer); writer.close(); fout.close(); FSDataInputStream fin = fs.open(mFile); Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile), this.fs.getFileStatus(mFile).getLen(), cacheConf, conf); reader.loadFileInfo(); // No data -- this should return false. assertFalse(reader.getScanner(false, false).seekTo()); someReadingWithMetaBlock(reader); fs.delete(mFile, true); reader.close(); fin.close(); }
@Test public void testNullMetaBlocks() throws Exception { for (Compression.Algorithm compressAlgo : HBaseCommonTestingUtility.COMPRESSION_ALGORITHMS) { Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile"); FSDataOutputStream fout = createFSOutput(mFile); HFileContext meta = new HFileContextBuilder().withCompression(compressAlgo) .withBlockSize(minBlockSize).build(); Writer writer = HFile.getWriterFactory(conf, cacheConf) .withOutputStream(fout) .withFileContext(meta) .create(); KeyValue kv = new KeyValue("foo".getBytes(), "f1".getBytes(), null, "value".getBytes()); writer.append(kv); writer.close(); fout.close(); Reader reader = HFile.createReader(fs, mFile, cacheConf, true, conf); reader.loadFileInfo(); assertNull(reader.getMetaBlock("non-existant", false)); } }
/** * Create a truncated hfile and verify that exception thrown. */ @Test public void testCorruptTruncatedHFile() throws IOException { Path f = new Path(ROOT_DIR, testName.getMethodName()); HFileContext context = new HFileContextBuilder().build(); Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f) .withFileContext(context).create(); writeSomeRecords(w, 0, 100, false); w.close(); Path trunc = new Path(f.getParent(), "trucated"); truncateFile(fs, w.getPath(), trunc); try { Reader r = HFile.createReader(fs, trunc, cacheConf, true, conf); } catch (CorruptHFileException che) { // Expected failure return; } fail("Should have thrown exception"); }
/** * Test empty HFile. * Test all features work reasonably when hfile is empty of entries. * @throws IOException */ @Test public void testEmptyHFile() throws IOException { Path f = new Path(ROOT_DIR, testName.getMethodName()); HFileContext context = new HFileContextBuilder().withIncludesTags(false).build(); Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).withFileContext(context).create(); w.close(); Reader r = HFile.createReader(fs, f, cacheConf, true, conf); r.loadFileInfo(); assertFalse(r.getFirstKey().isPresent()); assertFalse(r.getLastKey().isPresent()); }
.withCompression(HFileWriterImpl.compressionByName(codec)) .build(); Writer writer = HFile.getWriterFactory(conf, cacheConf) .withOutputStream(fout) .withFileContext(meta)
.withDataBlockEncoding(encoding) .build(); HFile.Writer writer = HFile.getWriterFactory(configuration, new CacheConfig(configuration)) .withPath(fs, path) .withFileContext(meta)
HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build(); HFile.Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(fs, p).withFileContext(meta).create();
private void createHFile(Path path, byte[] family, byte[] qualifier, byte[] startKey, byte[] endKey, int numRows) throws IOException { HFile.Writer writer = null; long now = System.currentTimeMillis(); try { HFileContext context = new HFileContextBuilder().build(); writer = HFile.getWriterFactory(conf, new CacheConfig(conf)).withPath(fs, path) .withFileContext(context).create(); // subtract 2 since numRows doesn't include boundary keys for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, true, numRows - 2)) { KeyValue kv = new KeyValue(key, family, qualifier, now, key); writer.append(kv); } } finally { if (writer != null) { writer.close(); } } }
.withDataBlockEncoding(DataBlockEncoding.NONE).build(); HFile.Writer writer = HFile.getWriterFactory(conf, cacheConf).withPath(fs, hfilePath).withFileContext(meta) .create(); Random rand = new Random(19231737);
/** * Create an HFile with the given number of rows with a specified value. */ public static void createHFile(FileSystem fs, Path path, byte[] family, byte[] qualifier, byte[] value, int numRows) throws IOException { HFileContext context = new HFileContextBuilder().withBlockSize(BLOCKSIZE) .withCompression(COMPRESSION) .build(); HFile.Writer writer = HFile .getWriterFactory(conf, new CacheConfig(conf)) .withPath(fs, path) .withFileContext(context) .create(); long now = System.currentTimeMillis(); try { // subtract 2 since iterateOnSplits doesn't include boundary keys for (int i = 0; i < numRows; i++) { KeyValue kv = new KeyValue(rowkey(i), family, qualifier, now, value); writer.append(kv); } writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(now)); } finally { writer.close(); } }
HFileContext meta = new HFileContextBuilder().withBlockSize(1024).build(); HFile.Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(fs, p).withFileContext(meta).create();
HFile.Writer writer = HFile.getWriterFactory(conf, cacheConf) .withOutputStream(out) .withFileContext(fileContext)
TEST_UTIL.getRandomUUID().toString() + ".hfile"); FSDataOutputStream out = fs.create(path); HFile.Writer writer = HFile.getWriterFactory(conf, cacheConf) .withOutputStream(out) .withFileContext(fileContext)
.build(); HFile.Writer writer = HFile.getWriterFactory(conf, cacheConf) .withPath(fs, hfilePath) .withFileContext(meta)
/** * Returns the factory to be used to create {@link HFile} writers. * Disables block cache access for all writers created through the * returned factory. */ public static final WriterFactory getWriterFactoryNoCache(Configuration conf) { Configuration tempConf = new Configuration(conf); tempConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.0f); return HFile.getWriterFactory(conf, new CacheConfig(tempConf)); }