private static Path getManifestPath(Configuration conf, Path backupRootPath, String backupId) throws IOException { FileSystem fs = backupRootPath.getFileSystem(conf); Path manifestPath = new Path(getBackupPath(backupRootPath.toString(), backupId) + Path.SEPARATOR + BackupManifest.MANIFEST_FILE_NAME); if (!fs.exists(manifestPath)) { String errorMsg = "Could not find backup manifest " + BackupManifest.MANIFEST_FILE_NAME + " for " + backupId + ". File " + manifestPath + " does not exists. Did " + backupId + " correspond to previously taken backup ?"; throw new IOException(errorMsg); } return manifestPath; }
/** * Get backup tmp directory for backupId * @param backupRoot backup root * @param backupId backup id * @return backup tmp directory path */ public static Path getBackupTmpDirPathForBackupId(String backupRoot, String backupId) { return new Path(getBackupTmpDirPath(backupRoot), backupId); }
public static Path getLogBackupPath(String backupRootDir, String backupId) { return new Path(getLogBackupDir(backupRootDir, backupId)); }
FileSystem fs = FileSystem.get(new Path(backupRoot).toUri(), new Configuration()); String backupId = BackupUtils.findMostRecentBackupId(backupIds); Path tmpPath = HBackupFileSystem.getBackupTmpDirPathForBackupId(backupRoot, backupId); if (fs.exists(tmpPath)) { Path destPath = HBackupFileSystem.getBackupPath(backupRoot, backupId); if (!fs.delete(destPath, true)) { System.out.println("Failed to delete " + destPath);
Path backupRoot = new Path(rootDir); RestoreTool restoreTool = new RestoreTool(conf, backupRoot, backupId); Path tableBackupPath = HBackupFileSystem.getTableBackupPath(sTable, backupRoot, backupId); String lastIncrBackupId = images.length == 1 ? null : images[images.length - 1].getBackupId(); BackupManifest manifest = HBackupFileSystem.getManifest(conf, backupRoot, backupId); if (manifest.getType() == BackupType.FULL) { LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from full" + " backup image " BackupImage im = images[i]; String fileBackupDir = HBackupFileSystem.getTableBackupDir(im.getRootDir(), im.getBackupId(), sTable); List<Path> list = getFilesRecursively(fileBackupDir); dirList.addAll(list);
public static String getTableBackupDataDir(String backupRootDir, String backupId, TableName tableName) { return getTableBackupDir(backupRootDir, backupId, tableName) + Path.SEPARATOR + "data"; }
/** * Check whether the backup image path and there is manifest file in the path. * @param backupManifestMap If all the manifests are found, then they are put into this map * @param tableArray the tables involved * @throws IOException exception */ public static void checkImageManifestExist(HashMap<TableName, BackupManifest> backupManifestMap, TableName[] tableArray, Configuration conf, Path backupRootPath, String backupId) throws IOException { for (TableName tableName : tableArray) { BackupManifest manifest = getManifest(conf, backupRootPath, backupId); backupManifestMap.put(tableName, manifest); } } }
/** * Returns value represent path for path to backup table snapshot directory: * "/$USER/SBACKUP_ROOT/backup_id/namespace/table/.hbase-snapshot" * @param backupRootPath backup root path * @param tableName table name * @param backupId backup Id * @return path for snapshot */ Path getTableSnapshotPath(Path backupRootPath, TableName tableName, String backupId) { return new Path(HBackupFileSystem.getTableBackupPath(tableName, backupRootPath, backupId), HConstants.SNAPSHOT_DIR_NAME); }
public void execute() throws IOException { // case VALIDATION: // check the target tables checkTargetTables(tTableArray, isOverwrite); // case RESTORE_IMAGES: HashMap<TableName, BackupManifest> backupManifestMap = new HashMap<>(); // check and load backup image manifest for the tables Path rootPath = new Path(targetRootDir); HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray, conf, rootPath, backupId); restore(backupManifestMap, sTableArray, tTableArray, isOverwrite); } }
public static BackupManifest getManifest(Configuration conf, Path backupRootPath, String backupId) throws IOException { BackupManifest manifest = new BackupManifest(conf, getManifestPath(conf, backupRootPath, backupId)); return manifest; }
Path tmpBackupDir = HBackupFileSystem.getBackupTmpDirPathForBackupId(backupRoot, mergedBackupId); Path backupDirPath = HBackupFileSystem.getBackupPath(backupRoot, mergedBackupId);
/** * Given the backup root dir, backup id and the table name, return the backup image location, * which is also where the backup manifest file is. return value look like: * "hdfs://backup.hbase.org:9000/user/biadmin/backup/backup_1396650096738/default/t1_dn/", where * "hdfs://backup.hbase.org:9000/user/biadmin/backup" is a backup root directory * @param backupRootPath backup root path * @param tableName table name * @param backupId backup Id * @return backupPath for the particular table */ public static Path getTableBackupPath(TableName tableName, Path backupRootPath, String backupId) { return new Path(getTableBackupDir(backupRootPath.toString(), backupId, tableName)); }
protected void updateBackupManifest(String backupRoot, String mergedBackupId, List<String> backupsToDelete) throws IllegalArgumentException, IOException { BackupManifest manifest = HBackupFileSystem.getManifest(conf, new Path(backupRoot), mergedBackupId); manifest.getBackupImage().removeAncestors(backupsToDelete); // save back manifest.store(conf); }
/** * return value represent path for: * ".../user/biadmin/backup1/default/t1_dn/backup_1396650096738/archive/data/default/t1_dn" * @param tableName table name * @return path to table archive * @throws IOException exception */ Path getTableArchivePath(TableName tableName) throws IOException { Path baseDir = new Path(HBackupFileSystem.getTableBackupPath(tableName, backupRootPath, backupId), HConstants.HFILE_ARCHIVE_DIRECTORY); Path dataDir = new Path(baseDir, HConstants.BASE_NAMESPACE_DIR); Path archivePath = new Path(dataDir, tableName.getNamespaceAsString()); Path tableArchivePath = new Path(archivePath, tableName.getQualifierAsString()); if (!fs.exists(tableArchivePath) || !fs.getFileStatus(tableArchivePath).isDirectory()) { LOG.debug("Folder tableArchivePath: " + tableArchivePath.toString() + " does not exists"); tableArchivePath = null; // empty table has no archive } return tableArchivePath; }
@Override public void restore(RestoreRequest request) throws IOException { if (request.isCheck()) { HashMap<TableName, BackupManifest> backupManifestMap = new HashMap<>(); // check and load backup image manifest for the tables Path rootPath = new Path(request.getBackupRootDir()); String backupId = request.getBackupId(); TableName[] sTableArray = request.getFromTables(); HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray, conn.getConfiguration(), rootPath, backupId); // Check and validate the backup image and its dependencies if (BackupUtils.validate(backupManifestMap, conn.getConfiguration())) { LOG.info(CHECK_OK); } else { LOG.error(CHECK_FAILED); } return; } // Execute restore request new RestoreTablesClient(conn, request).execute(); }
Path tmpBackupDir = HBackupFileSystem.getBackupTmpDirPathForBackupId(backupRoot, mergedBackupId); Path backupDirPath = HBackupFileSystem.getBackupPath(backupRoot, mergedBackupId); if (!fs.rename(backupDirPath, tmpBackupDir)) { throw new IOException("Failed to rename "+ backupDirPath +" to "+tmpBackupDir);
private static void checkRemoveBackupImages(FileSystem fs, String backupRoot, String[] backupIds) throws IOException { String mergedBackupId = BackupUtils.findMostRecentBackupId(backupIds); for (String backupId: backupIds) { if (backupId.equals(mergedBackupId)) { continue; } Path path = HBackupFileSystem.getBackupPath(backupRoot, backupId); if (fs.exists(path)) { if (!fs.delete(path, true)) { System.out.println("MERGE repair removing: "+ path +" - FAILED"); } else { System.out.println("MERGE repair removing: "+ path +" - OK"); } } } }
protected Path[] findInputDirectories(FileSystem fs, String backupRoot, TableName tableName, String[] backupIds) throws IOException { List<Path> dirs = new ArrayList<>(); for (String backupId : backupIds) { Path fileBackupDirPath = new Path(HBackupFileSystem.getTableBackupDir(backupRoot, backupId, tableName)); if (fs.exists(fileBackupDirPath)) { dirs.add(fileBackupDirPath); } else { if (LOG.isDebugEnabled()) { LOG.debug("File: " + fileBackupDirPath + " does not exist."); } } } Path[] ret = new Path[dirs.size()]; return dirs.toArray(ret); } }
protected void deleteBackupImages(List<String> backupIds, Connection conn, FileSystem fs, String backupRoot) throws IOException { // Delete from backup system table try (BackupSystemTable table = new BackupSystemTable(conn)) { for (String backupId : backupIds) { table.deleteBackupInfo(backupId); } } // Delete from file system for (String backupId : backupIds) { Path backupDirPath = HBackupFileSystem.getBackupPath(backupRoot, backupId); if (!fs.delete(backupDirPath, true)) { LOG.warn("Could not delete " + backupDirPath); } } }
protected void moveData(FileSystem fs, String backupRoot, Path bulkOutputPath, TableName tableName, String mergedBackupId) throws IllegalArgumentException, IOException { Path dest = new Path(HBackupFileSystem.getTableBackupDir(backupRoot, mergedBackupId, tableName)); FileStatus[] fsts = fs.listStatus(bulkOutputPath); for (FileStatus fst : fsts) { if (fst.isDirectory()) { String family = fst.getPath().getName(); Path newDst = new Path(dest, family); if (fs.exists(newDst)) { if (!fs.delete(newDst, true)) { throw new IOException("failed to delete :"+ newDst); } } else { fs.mkdirs(dest); } boolean result = fs.rename(fst.getPath(), dest); LOG.debug("MoveData from "+ fst.getPath() +" to "+ dest+" result="+ result); } } }