public static SnapshotManifest getSnapshotManifest(Configuration conf, String snapshotName, Path rootDir, FileSystem fs) throws IOException { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); return SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); }
conf.get(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR) != null; Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot); Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot, destConf); Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot); Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir; Path needSetOwnerDir = SnapshotDescriptionUtils.getSnapshotRootDir(outputRoot); if (outputFs.exists(needSetOwnerDir)) { if (skipTmp) { needSetOwnerDir = outputSnapshotDir; } else { needSetOwnerDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(outputRoot, destConf); if (outputFs.exists(needSetOwnerDir)) { needSetOwnerDir = snapshotTmpDir; SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir) .toBuilder() .setName(targetName) .build(); SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, initialOutputSnapshotDir, outputFs); if (filesUser != null || filesGroup != null) { outputFs.setOwner(new Path(initialOutputSnapshotDir,
/** * Returns the list of available snapshots in the specified location * @param conf the {@link Configuration} to use * @return the list of snapshots */ public static List<SnapshotDescription> getSnapshotList(final Configuration conf) throws IOException { Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = FileSystem.get(rootDir.toUri(), conf); Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); FileStatus[] snapshots = fs.listStatus(snapshotDir, new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs)); List<SnapshotDescription> snapshotLists = new ArrayList<>(snapshots.length); for (FileStatus snapshotDirStat: snapshots) { SnapshotProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()); snapshotLists.add(ProtobufUtil.createSnapshotDesc(snapshotDesc)); } return snapshotLists; }
/** * Get the directory to build a snapshot, before it is finalized * @param snapshotName name of the snapshot * @param rootDir root directory of the hbase installation * @param conf Configuration of the HBase instance * @return {@link Path} where one can build a snapshot */ public static Path getWorkingSnapshotDir(String snapshotName, final Path rootDir, Configuration conf) { return getSpecifiedSnapshotDir(getWorkingSnapshotDir(rootDir, conf), snapshotName); }
/** * Get the directory for a completed snapshot. This directory is a sub-directory of snapshot root * directory and all the data files for a snapshot are kept under this directory. * @param snapshotName name of the snapshot being taken * @param rootDir hbase root directory * @return the final directory for the completed snapshot */ public static Path getCompletedSnapshotDir(final String snapshotName, final Path rootDir) { return getSpecifiedSnapshotDir(getSnapshotsDir(rootDir), snapshotName); }
/** * Returns the store file names in the snapshot. * * @param conf The current {@link Configuration} instance. * @param fs {@link FileSystem} * @param snapshotDir {@link Path} to the Snapshot directory * @throws IOException if an error occurred while scanning the directory * @return the names of hfiles in the specified snaphot */ public static Set<String> getHFileNames(final Configuration conf, final FileSystem fs, final Path snapshotDir) throws IOException { SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); return getHFileNames(conf, fs, snapshotDir, desc); }
/** * Get the directory to build a snapshot, before it is finalized * @param snapshotName name of the snapshot * @param rootDir root directory of the hbase installation * @return {@link Path} where one can build a snapshot */ public static Path getWorkingSnapshotDir(String snapshotName, final Path rootDir) { return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshotName); }
/** * Get the directory for a specified snapshot. This directory is a sub-directory of snapshot root * directory and all the data files for a snapshot are kept under this directory. * @param snapshot snapshot being taken * @param rootDir hbase root directory * @return the final directory for the completed snapshot */ public static Path getCompletedSnapshotDir(final SnapshotDescription snapshot, final Path rootDir) { return getCompletedSnapshotDir(snapshot.getName(), rootDir); }
/** * Get the directory to build a snapshot, before it is finalized * @param snapshot snapshot that will be built * @param rootDir root directory of the hbase installation * @param conf Configuration of the HBase instance * @return {@link Path} where one can build a snapshot */ public static Path getWorkingSnapshotDir(SnapshotDescription snapshot, final Path rootDir, Configuration conf) { return getWorkingSnapshotDir(snapshot.getName(), rootDir, conf); }
private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName, final int numRegions, final int version) throws IOException { TableDescriptor htd = createHtd(tableName); RegionData[] regions = createTable(htd, numRegions); SnapshotProtos.SnapshotDescription desc = SnapshotProtos.SnapshotDescription.newBuilder() .setTable(htd.getTableName().getNameAsString()) .setName(snapshotName) .setVersion(version) .build(); Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir, conf); SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs); return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions); }
this.conf = this.master.getConfiguration(); this.rootDir = this.master.getMasterFileSystem().getRootDir(); this.workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir, conf); Preconditions.checkArgument(!SnapshotDescriptionUtils.isSubDirectoryOf(workingDir, rootDir) || SnapshotDescriptionUtils.isWithinDefaultWorkingDir(workingDir, conf), "The working directory " + workingDir + " cannot be in the root directory unless it is " + "within the default working directory"); this.snapshotTable = TableName.valueOf(snapshot.getTable()); this.rootFs = this.master.getMasterFileSystem().getFileSystem(); this.snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir); this.workingDirFs = this.workingDir.getFileSystem(this.conf); this.monitor = new ForeignExceptionDispatcher(snapshot.getName());
/** * Get the default working directory for snapshots - where they are built, where they are * temporarily copied on export, etc. * @param rootDir root directory of the HBase installation * @return Path to the default snapshot tmp directory, relative to the passed root directory */ private static Path getDefaultWorkingSnapshotDir(final Path rootDir) { return new Path(getSnapshotsDir(rootDir), SNAPSHOT_TMP_DIR_NAME); }
/** * Get the directory for a completed snapshot. This directory is a sub-directory of snapshot root * directory and all the data files for a snapshot are kept under this directory. * @param snapshotName name of the snapshot being taken * @param rootDir hbase root directory * @return the final directory for the completed snapshot */ public static Path getCompletedSnapshotDir(final String snapshotName, final Path rootDir) { return getCompletedSnapshotDir(getSnapshotsDir(rootDir), snapshotName); }
public Path commit() throws IOException { ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName()); SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor); manifest.addTableDescriptor(htd); manifest.consolidate(); SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs); snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir); return snapshotDir; }
/** * Triggers an asynchronous attempt to take a snapshot. * {@inheritDoc} */ @Override public SnapshotResponse snapshot(RpcController controller, SnapshotRequest request) throws ServiceException { try { master.checkInitialized(); master.snapshotManager.checkSnapshotSupport(); LOG.info(master.getClientIdAuditPrefix() + " snapshot request for:" + ClientSnapshotDescriptionUtils.toString(request.getSnapshot())); // get the snapshot information SnapshotDescription snapshot = SnapshotDescriptionUtils.validate( request.getSnapshot(), master.getConfiguration()); master.snapshotManager.takeSnapshot(snapshot); // send back the max amount of time the client should wait for the snapshot to complete long waitTime = SnapshotDescriptionUtils.getMaxMasterTimeout(master.getConfiguration(), snapshot.getType(), SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME); return SnapshotResponse.newBuilder().setExpectedTimeout(waitTime).build(); } catch (ForeignException e) { throw new ServiceException(e.getCause()); } catch (IOException e) { throw new ServiceException(e); } }
SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, workingDirFs); snapshotManifest.addTableDescriptor(this.htd); monitor.rethrowException();
/** * Triggers an asynchronous attempt to take a snapshot. * {@inheritDoc} */ @Override public long snapshot(final HSnapshotDescription request) throws IOException { LOG.debug("Submitting snapshot request for:" + SnapshotDescriptionUtils.toString(request.getProto())); try { this.snapshotManager.checkSnapshotSupport(); } catch (UnsupportedOperationException e) { throw new IOException(e); } // get the snapshot information SnapshotDescription snapshot = SnapshotDescriptionUtils.validate(request.getProto(), this.conf); snapshotManager.takeSnapshot(snapshot); // send back the max amount of time the client should wait for the snapshot to complete long waitTime = SnapshotDescriptionUtils.getMaxMasterTimeout(conf, snapshot.getType(), SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME); return waitTime; }
SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, workingDir, this.fs); new TableInfoCopyTask(monitor, snapshot, fs, rootDir).call(); monitor.rethrowException(); String reason = "Failed taking snapshot " + SnapshotDescriptionUtils.toString(snapshot) + " due to exception:" + e.getMessage(); LOG.error(reason, e); monitor.receive(ee); cancel("Failed to take snapshot '" + SnapshotDescriptionUtils.toString(snapshot) + "' due to exception"); } finally {
@Test public void testValidateMissingTableName() throws IOException { Configuration conf = new Configuration(false); try { SnapshotDescriptionUtils.validate(SnapshotDescription.newBuilder().setName("fail").build(), conf); fail("Snapshot was considered valid without a table name"); } catch (IllegalArgumentException e) { LOG.debug("Correctly failed when snapshot doesn't have a tablename"); } }
/** * Create a snapshot timer for the master which notifies the monitor when an error occurs * @param snapshot snapshot to monitor * @param conf configuration to use when getting the max snapshot life * @param monitor monitor to notify when the snapshot life expires * @return the timer to use update to signal the start and end of the snapshot */ public static TimeoutExceptionInjector getMasterTimerAndBindToMonitor(SnapshotDescription snapshot, Configuration conf, ForeignExceptionListener monitor) { long maxTime = SnapshotDescriptionUtils.getMaxMasterTimeout(conf, snapshot.getType(), SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME); return new TimeoutExceptionInjector(monitor, maxTime); }