public static void concurrentVisitReferencedFiles(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest, final String desc, final StoreFileVisitor visitor) throws IOException { final Path snapshotDir = manifest.getSnapshotDir(); List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); if (regionManifests == null || regionManifests.isEmpty()) { LOG.debug("No manifest files present: " + snapshotDir); return; } ExecutorService exec = SnapshotManifest.createExecutor(conf, desc); try { concurrentVisitReferencedFiles(conf, fs, manifest, exec, visitor); } finally { exec.shutdown(); } }
/** * Verify the validity of the snapshot * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param manifest snapshot manifest to inspect * @throws CorruptedSnapshotException if the snapshot is corrupted * @throws IOException if an error occurred while scanning the directory */ public static void verifySnapshot(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest) throws IOException { final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription(); final Path snapshotDir = manifest.getSnapshotDir(); concurrentVisitReferencedFiles(conf, fs, manifest, "VerifySnapshot", new StoreFileVisitor() { @Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { verifyStoreFile(conf, fs, snapshotDir, snapshotDesc, regionInfo, family, storeFile); } }); }
/** * Returns the snapshot stats * @param conf the {@link Configuration} to use * @param snapshotDesc HBaseProtos.SnapshotDescription to get stats from * @param filesMap {@link Map} store files map for all snapshots, it may be null * @return the snapshot stats */ public static SnapshotStats getSnapshotStats(final Configuration conf, final SnapshotProtos.SnapshotDescription snapshotDesc, final Map<Path, Integer> filesMap) throws IOException { Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = FileSystem.get(rootDir.toUri(), conf); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); final SnapshotStats stats = new SnapshotStats(conf, fs, snapshotDesc); SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, "SnapshotsStatsAggregation", new SnapshotReferenceUtil.SnapshotVisitor() { @Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { if (!storeFile.hasReference()) { stats.addStoreFile(regionInfo, family, storeFile, filesMap); } }}); return stats; }
SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, exec, new SnapshotReferenceUtil.SnapshotVisitor() { @Override public void storeFile(final RegionInfo regionInfo, final String family,
final SnapshotDescription desc = ProtobufUtil.createSnapshotDesc(snapshotDesc); final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, desc); SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest, "SnapshotInfo", new SnapshotReferenceUtil.SnapshotVisitor() {
/** * Verify the validity of the snapshot * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param manifest snapshot manifest to inspect * @throws CorruptedSnapshotException if the snapshot is corrupted * @throws IOException if an error occurred while scanning the directory */ public static void verifySnapshot(final Configuration conf, final FileSystem fs, final SnapshotManifest manifest) throws IOException { final SnapshotDescription snapshotDesc = manifest.getSnapshotDescription(); final Path snapshotDir = manifest.getSnapshotDir(); concurrentVisitReferencedFiles(conf, fs, manifest, new StoreFileVisitor() { @Override public void storeFile(final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { verifyStoreFile(conf, fs, snapshotDir, snapshotDesc, regionInfo, family, storeFile); } }); }
/** * Returns the snapshot stats * @param conf the {@link Configuration} to use * @param snapshot {@link SnapshotDescription} to get stats from * @return the snapshot stats */ public static SnapshotStats getSnapshotStats(final Configuration conf, final SnapshotDescription snapshot) throws IOException { Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = FileSystem.get(rootDir.toUri(), conf); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot); final SnapshotStats stats = new SnapshotStats(conf, fs, snapshot); SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, new SnapshotReferenceUtil.SnapshotVisitor() { @Override public void storeFile(final HRegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { if (!storeFile.hasReference()) { stats.addStoreFile(regionInfo, family, storeFile); } } @Override public void logFile (final String server, final String logfile) throws IOException { stats.addLogFile(server, logfile); } }); return stats; }
final String table = snapshotDesc.getTable(); final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, snapshotDesc); SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest, new SnapshotReferenceUtil.SnapshotVisitor() { @Override