/** * Get the directory for a specified snapshot. This directory is a sub-directory of snapshot root * directory and all the data files for a snapshot are kept under this directory. * @param snapshot snapshot being taken * @param rootDir hbase root directory * @return the final directory for the completed snapshot */ public static Path getCompletedSnapshotDir(final SnapshotDescription snapshot, final Path rootDir) { return getCompletedSnapshotDir(snapshot.getName(), rootDir); }
public static SnapshotManifest getSnapshotManifest(Configuration conf, String snapshotName, Path rootDir, FileSystem fs) throws IOException { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); return SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); }
private void openWithoutRestoringSnapshot() throws IOException { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests(); if (regionManifests == null) { throw new IllegalArgumentException("Snapshot seems empty, snapshotName: " + snapshotName); } regions = new ArrayList<>(regionManifests.size()); regionManifests.stream().map(r -> HRegionInfo.convert(r.getRegionInfo())) .filter(this::isValidRegion).sorted().forEach(r -> regions.add(r)); htd = manifest.getTableDescriptor(); }
/** * Move the finished snapshot to its final, publicly visible directory - this marks the snapshot * as 'complete'. * @param snapshot description of the snapshot being tabken * @param rootdir root directory of the hbase installation * @param workingDir directory where the in progress snapshot was built * @param fs {@link FileSystem} where the snapshot was built * @throws org.apache.hadoop.hbase.snapshot.SnapshotCreationException if the * snapshot could not be moved * @throws IOException the filesystem could not be reached */ public static void completeSnapshot(SnapshotDescription snapshot, Path rootdir, Path workingDir, FileSystem fs) throws SnapshotCreationException, IOException { Path finishedDir = getCompletedSnapshotDir(snapshot, rootdir); LOG.debug("Snapshot is done, just moving the snapshot from " + workingDir + " to " + finishedDir); if (!fs.rename(workingDir, finishedDir)) { throw new SnapshotCreationException( "Failed to move working directory(" + workingDir + ") to completed directory(" + finishedDir + ").", ProtobufUtil.createSnapshotDesc(snapshot)); } }
/** * Load snapshot info and table descriptor for the specified snapshot * @param snapshotName name of the snapshot to load * @return false if snapshot is not found */ private boolean loadSnapshotInfo(final String snapshotName) throws IOException { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); if (!fs.exists(snapshotDir)) { LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir); return false; } SnapshotProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc); return true; }
/** * Check to see if the snapshot is one of the currently completed snapshots * Returns true if the snapshot exists in the "completed snapshots folder". * * @param snapshot expected snapshot to check * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is * not stored * @throws IOException if the filesystem throws an unexpected exception, * @throws IllegalArgumentException if snapshot name is invalid. */ private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException { try { final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); FileSystem fs = master.getMasterFileSystem().getFileSystem(); // check to see if the snapshot already exists return fs.exists(snapshotDir); } catch (IllegalArgumentException iae) { throw new UnknownSnapshotException("Unexpected exception thrown", iae); } }
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
/** * Returns the snapshot stats * @param conf the {@link Configuration} to use * @param snapshotDesc HBaseProtos.SnapshotDescription to get stats from * @param filesMap {@link Map} store files map for all snapshots, it may be null * @return the snapshot stats */ public static SnapshotStats getSnapshotStats(final Configuration conf, final SnapshotProtos.SnapshotDescription snapshotDesc, final Map<Path, Integer> filesMap) throws IOException { Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = FileSystem.get(rootDir.toUri(), conf); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); final SnapshotStats stats = new SnapshotStats(conf, fs, snapshotDesc); SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, "SnapshotsStatsAggregation", new SnapshotReferenceUtil.SnapshotVisitor() { @Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { if (!storeFile.hasReference()) { stats.addStoreFile(regionInfo, family, storeFile, filesMap); } }}); return stats; }
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( snapshotName, FSUtils.getRootDir(conf)); SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
@Test public void testFindsSnapshotFilesWhenCleaning() throws IOException { Configuration conf = TEST_UTIL.getConfiguration(); FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); Path rootDir = FSUtils.getRootDir(conf); Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY); FileSystem fs = FileSystem.get(conf); SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner(); cleaner.setConf(conf); // write an hfile to the snapshot directory String snapshotName = "snapshot"; byte[] snapshot = Bytes.toBytes(snapshotName); final TableName tableName = TableName.valueOf(name.getMethodName()); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); HRegionInfo mockRegion = new HRegionInfo(tableName); Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName()); Path familyDir = new Path(regionSnapshotDir, "family"); // create a reference to a supposedly valid hfile String hfile = "fd1e73e8a96c486090c5cec07b4894c4"; Path refFile = new Path(familyDir, hfile); // make sure the reference file exists fs.create(refFile); // create the hfile in the archive fs.mkdirs(archivedHfileDir); fs.createNewFile(new Path(archivedHfileDir, hfile)); // make sure that the file isn't deletable assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile))); }
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException { HashSet<String> files = new HashSet<>(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir( snapshotName, FSUtils.getRootDir(conf)); SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo( fs, snapshotDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd); // For each region referenced by the snapshot for (SnapshotRegionManifest rm : manifest.getRegionManifests()) { // For each column family in this region for (FamilyFiles ff : rm.getFamilyFilesList()) { // And each store file in that family for (StoreFile sf : ff.getStoreFilesList()) { files.add(sf.getName()); } } } return files; }
this.snapshotTable = TableName.valueOf(snapshot.getTable()); this.rootFs = this.master.getMasterFileSystem().getFileSystem(); this.snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); this.workingDirFs = this.workingDir.getFileSystem(this.conf); this.monitor = new ForeignExceptionDispatcher(snapshot.getName());
final boolean restoreAcl) throws IOException { FileSystem fs = master.getMasterFileSystem().getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);
public Path commit() throws IOException { ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName()); SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); manifest.addTableDescriptor(htd); manifest.consolidate(); SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs); snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir); return snapshotDir; }
/** * Action before cloning from snapshot. * @param env MasterProcedureEnv * @throws IOException * @throws InterruptedException */ private void preCloneSnapshot(final MasterProcedureEnv env) throws IOException, InterruptedException { if (!getTableName().isSystemTable()) { // Check and update namespace quota final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem(); SnapshotManifest manifest = SnapshotManifest.open( env.getMasterConfiguration(), mfs.getFileSystem(), SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()), snapshot); ProcedureSyncWait.getMasterQuotaManager(env) .checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size()); } final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost(); if (cpHost != null) { cpHost.preCreateTableAction(tableDescriptor, null, getUser()); } }
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshot); RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
final FileSystem fs = mfs.getFileSystem(); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, mfs.getRootDir()); SnapshotProtos.SnapshotDescription snapshotDesc =
Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); SnapshotManifest manifest = SnapshotManifest.open( env.getMasterServices().getConfiguration(), fs, snapshotDir, snapshot);
env.getMasterConfiguration(), mfs.getFileSystem(), SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()), snapshot); int snapshotRegionCount = manifest.getRegionManifestsMap().size();