@Override public String toStringDetailed() { StringBuilder sb = new StringBuilder(); sb.append(this.getPath().toString()); sb.append(", isReference=").append(isReference()); sb.append(", isBulkLoadResult=").append(isBulkLoadResult()); if (isBulkLoadResult()) { sb.append(", bulkLoadTS="); OptionalLong bulkLoadTS = getBulkLoadTimestamp(); if (bulkLoadTS.isPresent()) { sb.append(bulkLoadTS.getAsLong()); } else { sb.append("NotPresent"); } } else { sb.append(", seqid=").append(getMaxSequenceId()); } sb.append(", majorCompaction=").append(isMajorCompactionResult()); return sb.toString(); }
/** * Validates a mob file by opening and closing it. * @param conf The current configuration. * @param fs The current file system. * @param path The path where the mob file is saved. * @param cacheConfig The current cache config. */ private static void validateMobFile(Configuration conf, FileSystem fs, Path path, CacheConfig cacheConfig, boolean primaryReplica) throws IOException { HStoreFile storeFile = null; try { storeFile = new HStoreFile(fs, path, conf, cacheConfig, BloomType.NONE, primaryReplica); storeFile.initReader(); } catch (IOException e) { LOG.error("Failed to open mob file[" + path + "], keep it in temp directory.", e); throw e; } finally { if (storeFile != null) { storeFile.closeStoreFile(false); } } }
@Override public boolean apply(HStoreFile input) { return input.excludeFromMinorCompaction(); } }));
/** * Closes the readers of store files. * @param storeFiles The store files to be closed. */ private void closeStoreFileReaders(List<HStoreFile> storeFiles) { for (HStoreFile storeFile : storeFiles) { try { storeFile.closeStoreFile(true); } catch (IOException e) { LOG.warn("Failed to close the reader on store file " + storeFile.getPath(), e); } } }
/** * Initialize the reader used for pread. */ public void initReader() throws IOException { if (reader == null) { try { open(); } catch (Exception e) { try { boolean evictOnClose = cacheConf != null ? cacheConf.shouldEvictOnClose() : true; this.closeStoreFile(evictOnClose); } catch (IOException ee) { LOG.warn("failed to close reader", ee); } throw e; } } }
private List<Cell> prepareListOfTestSeeks(Path path) throws IOException { List<Cell> allKeyValues = new ArrayList<>(); // read all of the key values HStoreFile storeFile = new HStoreFile(testingUtility.getTestFileSystem(), path, configuration, cacheConf, BloomType.NONE, true); storeFile.initReader(); StoreFileReader reader = storeFile.getReader(); StoreFileScanner scanner = reader.getStoreFileScanner(true, false, false, 0, 0, false); Cell current; scanner.seek(KeyValue.LOWESTKEY); while (null != (current = scanner.next())) { allKeyValues.add(current); } storeFile.closeStoreFile(cacheConf.shouldEvictOnClose()); // pick seeks by random List<Cell> seeks = new ArrayList<>(); for (int i = 0; i < numberOfSeeks; ++i) { Cell keyValue = allKeyValues.get( randomizer.nextInt(allKeyValues.size())); seeks.add(keyValue); } clearBlockCache(); return seeks; }
for (Path newDelPath : delPartition.listDelFiles()) { HStoreFile sf = new HStoreFile(fs, newDelPath, conf, this.cacheConfig, BloomType.NONE, true); sf.initReader(); delPartition.addStoreFile(sf); affectedPartitions += delFiles.size(); assertTrue(Bytes.compareTo(partition.getStartKey(), CellUtil.cloneRow(delFiles.get(0).getLastKey().get())) <= 0); assertTrue(Bytes.compareTo(partition.getEndKey(), CellUtil.cloneRow(delFiles.get(delFiles.size() - 1).getFirstKey().get())) >= 0); for (HStoreFile storeFile : delPartition.getStoreFiles()) { try { storeFile.closeStoreFile(true); } catch (IOException e) { LOG.warn("Failed to close the reader on store file " + storeFile.getPath(), e);
/** * Create a mock StoreFile with the given attributes. */ private HStoreFile mockStoreFile(boolean bulkLoad, long size, long bulkTimestamp, long seqId, String path) { HStoreFile mock = Mockito.mock(HStoreFile.class); StoreFileReader reader = Mockito.mock(StoreFileReader.class); Mockito.doReturn(size).when(reader).length(); Mockito.doReturn(reader).when(mock).getReader(); Mockito.doReturn(bulkLoad).when(mock).isBulkLoadResult(); Mockito.doReturn(OptionalLong.of(bulkTimestamp)).when(mock).getBulkLoadTimestamp(); Mockito.doReturn(seqId).when(mock).getMaxSequenceId(); Mockito.doReturn(new Path(path)).when(mock).getPath(); String name = "mock storefile, bulkLoad=" + bulkLoad + " bulkTimestamp=" + bulkTimestamp + " seqId=" + seqId + " path=" + path; Mockito.doReturn(name).when(mock).toString(); return mock; }
private long countMobCellsInMetadata() throws IOException { long mobCellsCount = 0; Path mobDirPath = MobUtils.getMobFamilyPath(conf, htd.getTableName(), hcd.getNameAsString()); Configuration copyOfConf = new Configuration(conf); copyOfConf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f); CacheConfig cacheConfig = new CacheConfig(copyOfConf); if (fs.exists(mobDirPath)) { FileStatus[] files = UTIL.getTestFileSystem().listStatus(mobDirPath); for (FileStatus file : files) { HStoreFile sf = new HStoreFile(fs, file.getPath(), conf, cacheConfig, BloomType.NONE, true); sf.initReader(); Map<byte[], byte[]> fileInfo = sf.getReader().loadFileInfo(); byte[] count = fileInfo.get(MOB_CELLS_COUNT); assertTrue(count != null); mobCellsCount += Bytes.toLong(count); } } return mobCellsCount; }
f.initReader(); try { if (top) { Optional<Cell> lastKey = f.getLastKey(); if (f.getComparator().compare(splitKey, lastKey.get()) > 0) { return null; Optional<Cell> firstKey = f.getFirstKey(); if (f.getComparator().compare(splitKey, firstKey.get()) < 0) { return null; f.closeStoreFile(f.getCacheConf() != null ? f.getCacheConf().shouldEvictOnClose() : true); Path p = new Path(splitDir, f.getPath().getName() + "." + parentRegionName); return r.write(fs, p);
protected HStoreFile createMockStoreFile(final long sizeInBytes, final long seqId) { HStoreFile mockSf = mock(HStoreFile.class); StoreFileReader reader = mock(StoreFileReader.class); String stringPath = "/hbase/testTable/regionA/" + RandomStringUtils.random(FILENAME_LENGTH, 0, 0, true, true, null, random); Path path = new Path(stringPath); when(reader.getSequenceID()).thenReturn(seqId); when(reader.getTotalUncompressedBytes()).thenReturn(sizeInBytes); when(reader.length()).thenReturn(sizeInBytes); when(mockSf.getPath()).thenReturn(path); when(mockSf.excludeFromMinorCompaction()).thenReturn(false); when(mockSf.isReference()).thenReturn(false); // TODO come back to // this when selection takes this into account when(mockSf.getReader()).thenReturn(reader); String toString = MoreObjects.toStringHelper("MockStoreFile") .add("isReference", false) .add("fileSize", StringUtils.humanReadableInt(sizeInBytes)) .add("seqId", seqId) .add("path", stringPath).toString(); when(mockSf.toString()).thenReturn(toString); return mockSf; } }
@Override Path getPath() { return file.getPath(); } }
private static HStoreFile createFile() throws Exception { HStoreFile sf = mock(HStoreFile.class); when(sf.getMetadataValue(any())) .thenReturn(StripeStoreFileManager.INVALID_KEY); when(sf.getReader()).thenReturn(mock(StoreFileReader.class)); when(sf.getPath()).thenReturn(new Path("moo")); when(sf.getBulkLoadTimestamp()).thenReturn(OptionalLong.empty()); return sf; }
if(allFiles && (file.getModificationTimestamp() < oldestHFileTimestampToKeepMVCC)) { if(fd.minSeqIdToKeep < file.getMaxMemStoreTS()) { fd.minSeqIdToKeep = file.getMaxMemStoreTS(); long seqNum = file.getMaxSequenceId(); fd.maxSeqId = Math.max(fd.maxSeqId, seqNum); StoreFileReader r = file.getReader(); if (r == null) { LOG.warn("Null reader for " + file.getPath()); continue; LOG.debug("Compacting {}, keycount={}, bloomtype={}, size={}, " + "encoding={}, compression={}, seqNum={}{}", (file.getPath() == null? null: file.getPath().getName()), keyCount, r.getBloomFilterType().toString(),
private static HStoreFile createFile(long size) throws Exception { HStoreFile sf = mock(HStoreFile.class); when(sf.getPath()).thenReturn(new Path("moo")); StoreFileReader r = mock(StoreFileReader.class); when(r.getEntries()).thenReturn(size); when(r.length()).thenReturn(size); when(r.getBloomFilterType()).thenReturn(BloomType.NONE); when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class)); when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong(), anyLong(), anyBoolean())).thenReturn(mock(StoreFileScanner.class)); when(sf.getReader()).thenReturn(r); when(sf.getBulkLoadTimestamp()).thenReturn(OptionalLong.empty()); return sf; }
CacheConfig cacheConf = new CacheConfig(conf); FileSystem fs = FileSystem.get(conf); HStoreFile hsf = new HStoreFile(fs, path, conf, cacheConf, BloomType.NONE, true); hsf.initReader(); StoreFileReader reader = hsf.getReader(); reader.loadFileInfo(); KeyValueScanner scanner = reader.getStoreFileScanner(true, true, false, hsf.getMaxMemStoreTS(), 0, false); USE_TAG = reader.getHFileReader().getFileContext().isIncludesTags();
@Override public Collection<HStoreFile> getUnneededFiles(long maxTs, List<HStoreFile> filesCompacting) { ImmutableList<HStoreFile> files = storefiles; // 1) We can never get rid of the last file which has the maximum seqid. // 2) Files that are not the latest can't become one due to (1), so the rest are fair game. return files.stream().limit(Math.max(0, files.size() - 1)).filter(sf -> { long fileTs = sf.getReader().getMaxTimestamp(); if (fileTs < maxTs && !filesCompacting.contains(sf)) { LOG.info("Found an expired store file {} whose maxTimestamp is {}, which is below {}", sf.getPath(), fileTs, maxTs); return true; } else { return false; } }).collect(Collectors.toList()); }
HStore store = r.getStore(Bytes.toBytes(fn)); for (HStoreFile sf : store.getStorefiles()) { assertTrue(sf.toString().contains(fn)); assertTrue("Column family " + fn + " should have 3 copies", FSUtils.getDefaultReplication(TEST_UTIL.getTestFileSystem(), sf.getPath()) == (sf .getFileInfo().getFileStatus().getReplication())); assertTrue(sf.toString().contains(fn1)); assertTrue("Column family " + fn1 + " should have only 1 copy", 1 == sf.getFileInfo() .getFileStatus().getReplication());
/** * Computes the length of a store file without succumbing to any errors along the way. If an * error is encountered, the implementation returns {@code 0} instead of the actual size. * * @param file The file to compute the size of. * @return The size in bytes of the provided {@code file}. */ long getStoreFileSize(HStoreFile file) { long length = 0; try { file.initReader(); length = file.getReader().length(); } catch (IOException e) { LOG.trace("Failed to open reader when trying to compute store file size, ignoring", e); } finally { try { file.closeStoreFile( file.getCacheConf() != null ? file.getCacheConf().shouldEvictOnClose() : true); } catch (IOException e) { LOG.trace("Failed to close reader after computing store file size, ignoring", e); } } return length; }
private HStoreFile mockStoreFile(long createdTime) { StoreFileInfo info = mock(StoreFileInfo.class); when(info.getCreatedTimestamp()).thenReturn(createdTime); HStoreFile sf = mock(HStoreFile.class); when(sf.getReader()).thenReturn(mock(StoreFileReader.class)); when(sf.isHFile()).thenReturn(true); when(sf.getFileInfo()).thenReturn(info); return sf; }