@Override public Path getFilePath() { return reader.getHFileReader().getPath(); }
@Override public int compare(StoreFileScanner s1, StoreFileScanner s2) { Path p1 = s1.getReader().getHFileReader().getPath(); Path p2 = s2.getReader().getHFileReader().getPath(); long t1, t2; try { t1 = fs.getFileStatus(p1).getModificationTime(); t2 = fs.getFileStatus(p2).getModificationTime(); } catch (IOException ex) { throw new RuntimeException(ex); } return t1 < t2 ? -1 : t1 == t2 ? 1 : 0; } });
@Override public long getOldestHfileTs(boolean majorCompactionOnly) throws IOException { long result = Long.MAX_VALUE; for (HStore store : stores.values()) { Collection<HStoreFile> storeFiles = store.getStorefiles(); if (storeFiles == null) { continue; } for (HStoreFile file : storeFiles) { StoreFileReader sfReader = file.getReader(); if (sfReader == null) { continue; } HFile.Reader reader = sfReader.getHFileReader(); if (reader == null) { continue; } if (majorCompactionOnly) { byte[] val = reader.loadFileInfo().get(MAJOR_COMPACTION_KEY); if (val == null || !Bytes.toBoolean(val)) { continue; } } result = Math.min(result, reader.getFileContext().getFileCreateTime()); } } return result == Long.MAX_VALUE ? 0 : result; }
public static HStoreFile createDummyStoreFile(long maxSequenceId) throws Exception { // "Files" are totally unused, it's Scanner class below that gives compactor fake KVs. // But compaction depends on everything under the sun, so stub everything with dummies. HStoreFile sf = mock(HStoreFile.class); StoreFileReader r = mock(StoreFileReader.class); when(r.length()).thenReturn(1L); when(r.getBloomFilterType()).thenReturn(BloomType.NONE); when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class)); when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyLong(), anyBoolean())).thenReturn(mock(StoreFileScanner.class)); when(sf.getReader()).thenReturn(r); when(sf.getMaxSequenceId()).thenReturn(maxSequenceId); return sf; }
private static HStoreFile createFile(long size) throws Exception { HStoreFile sf = mock(HStoreFile.class); when(sf.getPath()).thenReturn(new Path("moo")); StoreFileReader r = mock(StoreFileReader.class); when(r.getEntries()).thenReturn(size); when(r.length()).thenReturn(size); when(r.getBloomFilterType()).thenReturn(BloomType.NONE); when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class)); when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyLong(), anyBoolean())).thenReturn(mock(StoreFileScanner.class)); when(sf.getReader()).thenReturn(r); when(sf.getBulkLoadTimestamp()).thenReturn(OptionalLong.empty()); return sf; }
if (hfileBloomType != cfBloomType) { LOG.info("HFile Bloom filter type for " + reader.getHFileReader().getName() + ": " + hfileBloomType + ", but " + cfBloomType + " specified in column family " + "configuration"); + reader.getHFileReader().getName());
r.getBloomFilterType().toString(), TraditionalBinaryPrefix.long2String(r.length(), "", 1), r.getHFileReader().getDataBlockEncoding(), compactionCompression, seqNum,
private void checkMobHFileEncrytption(Collection<HStoreFile> storefiles) { HStoreFile storeFile = storefiles.iterator().next(); HFile.Reader reader = storeFile.getReader().getHFileReader(); byte[] encryptionKey = reader.getTrailer().getEncryptionKey(); Assert.assertTrue(null != encryptionKey); Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName() .equals(HConstants.CIPHER_AES)); }
lastStoreFileReader.getHFileReader().getPath().toString()}); + lastStoreFileReader.getHFileReader().getName()); lastStoreFileReader.disableBloomFilterForTesting();
@Test public void testFlushedFileWithNoTags() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = new HTableDescriptor(tableName); htd.addFamily(new HColumnDescriptor(fam1)); HRegionInfo info = new HRegionInfo(tableName, null, null, false); Path path = TEST_UTIL.getDataTestDir(getClass().getSimpleName()); region = HBaseTestingUtility.createRegionAndWAL(info, path, TEST_UTIL.getConfiguration(), htd); Put put = new Put(Bytes.toBytes("a-b-0-0")); put.addColumn(fam1, qual1, Bytes.toBytes("c1-value")); region.put(put); region.flush(true); HStore store = region.getStore(fam1); Collection<HStoreFile> storefiles = store.getStorefiles(); for (HStoreFile sf : storefiles) { assertFalse("Tags should not be present " ,sf.getReader().getHFileReader().getFileContext().isIncludesTags()); } }
private boolean verifyEncryption(TableName tableName, String familyName) throws IOException { Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName); boolean hasFiles = false; if (fs.exists(mobDirPath)) { FileStatus[] files = fs.listStatus(mobDirPath); hasFiles = files != null && files.length > 0; Assert.assertTrue(hasFiles); Path path = files[0].getPath(); CacheConfig cacheConf = new CacheConfig(conf); HStoreFile sf = new HStoreFile(TEST_UTIL.getTestFileSystem(), path, conf, cacheConf, BloomType.NONE, true); sf.initReader(); HFile.Reader reader = sf.getReader().getHFileReader(); byte[] encryptionKey = reader.getTrailer().getEncryptionKey(); Assert.assertTrue(null != encryptionKey); Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName() .equals(HConstants.CIPHER_AES)); } return hasFiles; }
@Test public void testFlushedFileWithVisibilityTags() throws Exception { final byte[] qual2 = Bytes.toBytes("qual2"); TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); HTableDescriptor desc = new HTableDescriptor(tableName); HColumnDescriptor col = new HColumnDescriptor(fam); desc.addFamily(col); TEST_UTIL.getAdmin().createTable(desc); try (Table table = TEST_UTIL.getConnection().getTable(tableName)) { Put p1 = new Put(row1); p1.addColumn(fam, qual, value); p1.setCellVisibility(new CellVisibility(CONFIDENTIAL)); Put p2 = new Put(row1); p2.addColumn(fam, qual2, value); p2.setCellVisibility(new CellVisibility(SECRET)); RowMutations rm = new RowMutations(row1); rm.add(p1); rm.add(p2); table.mutateRow(rm); } TEST_UTIL.getAdmin().flush(tableName); List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName); HStore store = regions.get(0).getStore(fam); Collection<HStoreFile> storefiles = store.getStorefiles(); assertTrue(storefiles.size() > 0); for (HStoreFile storeFile : storefiles) { assertTrue(storeFile.getReader().getHFileReader().getFileContext().isIncludesTags()); } }
HStoreFile sf = new HStoreFile(fs, path, conf, cacheConf, BloomType.ROWCOL, true); sf.initReader(); HFile.Reader reader = sf.getReader().getHFileReader(); try {
KeyValueScanner scanner = reader.getStoreFileScanner(true, true, false, hsf.getMaxMemStoreTS(), 0, false); USE_TAG = reader.getHFileReader().getFileContext().isIncludesTags();
@Override public int compare(StoreFileScanner s1, StoreFileScanner s2) { Path p1 = s1.getReader().getHFileReader().getPath(); Path p2 = s2.getReader().getHFileReader().getPath(); long t1, t2; try { t1 = fs.getFileStatus(p1).getModificationTime(); t2 = fs.getFileStatus(p2).getModificationTime(); } catch (IOException ex) { throw new RuntimeException(ex); } return t1 < t2 ? -1 : t1 == t2 ? 1 : 0; } });
private static HStoreFile createFile(long size) throws Exception { HStoreFile sf = mock(HStoreFile.class); when(sf.getPath()).thenReturn(new Path("moo")); StoreFileReader r = mock(StoreFileReader.class); when(r.getEntries()).thenReturn(size); when(r.length()).thenReturn(size); when(r.getBloomFilterType()).thenReturn(BloomType.NONE); when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class)); when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyLong(), anyBoolean())).thenReturn(mock(StoreFileScanner.class)); when(sf.getReader()).thenReturn(r); when(sf.getBulkLoadTimestamp()).thenReturn(OptionalLong.empty()); return sf; }
public static HStoreFile createDummyStoreFile(long maxSequenceId) throws Exception { // "Files" are totally unused, it's Scanner class below that gives compactor fake KVs. // But compaction depends on everything under the sun, so stub everything with dummies. HStoreFile sf = mock(HStoreFile.class); StoreFileReader r = mock(StoreFileReader.class); when(r.length()).thenReturn(1L); when(r.getBloomFilterType()).thenReturn(BloomType.NONE); when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class)); when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyLong(), anyBoolean())).thenReturn(mock(StoreFileScanner.class)); when(sf.getReader()).thenReturn(r); when(sf.getMaxSequenceId()).thenReturn(maxSequenceId); return sf; }
private void checkMobHFileEncrytption(Collection<HStoreFile> storefiles) { HStoreFile storeFile = storefiles.iterator().next(); HFile.Reader reader = storeFile.getReader().getHFileReader(); byte[] encryptionKey = reader.getTrailer().getEncryptionKey(); Assert.assertTrue(null != encryptionKey); Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName() .equals(HConstants.CIPHER_AES)); }
@Test public void testFlushedFileWithNoTags() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = new HTableDescriptor(tableName); htd.addFamily(new HColumnDescriptor(fam1)); HRegionInfo info = new HRegionInfo(tableName, null, null, false); Path path = TEST_UTIL.getDataTestDir(getClass().getSimpleName()); region = HBaseTestingUtility.createRegionAndWAL(info, path, TEST_UTIL.getConfiguration(), htd); Put put = new Put(Bytes.toBytes("a-b-0-0")); put.addColumn(fam1, qual1, Bytes.toBytes("c1-value")); region.put(put); region.flush(true); HStore store = region.getStore(fam1); Collection<HStoreFile> storefiles = store.getStorefiles(); for (HStoreFile sf : storefiles) { assertFalse("Tags should not be present " ,sf.getReader().getHFileReader().getFileContext().isIncludesTags()); } }
private boolean verifyEncryption(TableName tableName, String familyName) throws IOException { Path mobDirPath = MobUtils.getMobFamilyPath(conf, tableName, familyName); boolean hasFiles = false; if (fs.exists(mobDirPath)) { FileStatus[] files = fs.listStatus(mobDirPath); hasFiles = files != null && files.length > 0; Assert.assertTrue(hasFiles); Path path = files[0].getPath(); CacheConfig cacheConf = new CacheConfig(conf); HStoreFile sf = new HStoreFile(TEST_UTIL.getTestFileSystem(), path, conf, cacheConf, BloomType.NONE, true); sf.initReader(); HFile.Reader reader = sf.getReader().getHFileReader(); byte[] encryptionKey = reader.getTrailer().getEncryptionKey(); Assert.assertTrue(null != encryptionKey); Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName() .equals(HConstants.CIPHER_AES)); } return hasFiles; }