void restoreSnapshot(Configuration conf, String snapshotName, Path rootDir, Path restoreDir, FileSystem fs) throws IOException { RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName); }
RestoreSnapshotHelper helper = new RestoreSnapshotHelper(conf, fs, manifest, manifest.getTableDescriptor(), restoreDir, monitor, status, false); RestoreMetaChanges metaChanges = helper.restoreHdfsRegions(); // TODO: parallelize.
List<RegionInfo> tableRegions = getTableRegions(); List<RegionInfo> mobRegions = new ArrayList<>(1); mobRegions.add(mobRegion); restoreHdfsMobRegions(exec, regionManifests, mobRegions); regionNames.remove(mobRegion.getEncodedName()); status.setStatus("Finished restoring mob region."); cloneHdfsMobRegion(regionManifests, mobRegion); regionNames.remove(mobRegion.getEncodedName()); status.setStatus("Finished cloning mob region."); RegionInfo[] clonedRegions = cloneHdfsRegions(exec, regionManifests, regionsToAdd); metaChanges.setNewRegions(clonedRegions); status.setStatus("Finished cloning regions."); restoreHdfsRegions(exec, regionManifests, metaChanges.getRegionsToRestore()); status.setStatus("Finished restoring all table regions."); removeHdfsRegions(exec, metaChanges.getRegionsToRemove()); status.setStatus("Finished deleting excess regions from table.");
final SnapshotRegionManifest regionManifest, Path regionDir) throws IOException { Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles = getRegionHFileReferences(regionManifest); Set<String> familyFiles = getTableRegionFamilyFiles(familyDir); List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName()); LOG.debug("Adding HFileLink " + storeFile.getName() + " to region=" + regionInfo.getEncodedName() + " table=" + tableName); restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
List<HRegionInfo> tableRegions = getTableRegions(); if (tableRegions != null) { monitor.rethrowException(); restoreHdfsRegions(metaChanges.getRegionsToRestore()); removeHdfsRegions(metaChanges.getRegionsToRemove()); HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd); metaChanges.setNewRegions(clonedRegions); restoreWALs();
/** * Restore the on-disk table to a specified snapshot state. * @return the set of regions touched by the restore operation */ public RestoreMetaChanges restoreHdfsRegions() throws IOException { ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "RestoreSnapshot"); try { return restoreHdfsRegions(exec); } finally { exec.shutdown(); } }
/** * Create a new {@link RegionInfo} from the snapshot region info. * Keep the same startKey, endKey, regionId and split information but change * the table name. * * @param snapshotRegionInfo Info for region to clone. * @return the new HRegion instance */ public RegionInfo cloneRegionInfo(final RegionInfo snapshotRegionInfo) { return cloneRegionInfo(tableDesc.getTableName(), snapshotRegionInfo); }
Set<String> familyFiles = getTableRegionFamilyFiles(familyDir); List<String> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName()); if (snapshotFamilyFiles != null) { LOG.trace("Adding HFileLink " + hfileName + " to region=" + regionInfo.getEncodedName() + " table=" + tableName); restoreStoreFile(familyDir, regionInfo, hfileName); restoreStoreFile(familyDir, regionInfo, hfileName);
/** * Initialize the restore helper, based on the snapshot and table information provided. */ private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir, final SnapshotDescription sd, final TableDescriptor htdClone) throws IOException { ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); MonitoredTask status = Mockito.mock(MonitoredTask.class); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd); return new RestoreSnapshotHelper(conf, fs, manifest, htdClone, rootDir, monitor, status); }
/** * Create a new {@link HFileLink} to reference the store file. * <p>The store file in the snapshot can be a simple hfile, an HFileLink or a reference. * <ul> * <li>hfile: abc -> table=region-abc * <li>reference: abc.1234 -> table=region-abc.1234 * <li>hfilelink: table=region-hfile -> table=region-hfile * </ul> * @param familyDir destination directory for the store file * @param regionInfo destination region info for the table * @param createBackRef - Whether back reference should be created. Defaults to true. * @param storeFile store file name (can be a Reference, HFileLink or simple HFile) */ private void restoreStoreFile(final Path familyDir, final RegionInfo regionInfo, final SnapshotRegionManifest.StoreFile storeFile, final boolean createBackRef) throws IOException { String hfileName = storeFile.getName(); if (HFileLink.isHFileLink(hfileName)) { HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef); } else if (StoreFileInfo.isReference(hfileName)) { restoreReferenceFile(familyDir, regionInfo, storeFile); } else { HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef); } }
/** * Clone region directory content from the snapshot info. * * Each region is encoded with the table name, so the cloned region will have * a different region name. * * Instead of copying the hfiles a HFileLink is created. * * @param regionDir {@link Path} cloned dir * @param snapshotRegionInfo */ private void cloneRegion(final Path regionDir, final RegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException { final String tableName = tableDesc.getTableName().getNameAsString(); for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) { Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) { LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName); restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); } } }
List<HRegionInfo> tableRegions = getTableRegions(); if (tableRegions != null) { monitor.rethrowException(); restoreHdfsRegions(exec, regionManifests, metaChanges.getRegionsToRestore()); status.setStatus("Finished restoring all table regions."); removeHdfsRegions(exec, metaChanges.getRegionsToRemove()); status.setStatus("Finished deleting excess regions from table."); HRegionInfo[] clonedRegions = cloneHdfsRegions(exec, regionManifests, regionsToAdd); metaChanges.setNewRegions(clonedRegions); status.setStatus("Finished cloning regions.");
final SnapshotRegionManifest regionManifest) throws IOException { Map<String, List<SnapshotRegionManifest.StoreFile>> snapshotFiles = getRegionHFileReferences(regionManifest); Set<String> familyFiles = getTableRegionFamilyFiles(familyDir); List<SnapshotRegionManifest.StoreFile> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName()); LOG.debug("Adding HFileLink " + storeFile.getName() + " to region=" + regionInfo.getEncodedName() + " table=" + tableName); restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs); restoreStoreFile(familyDir, regionInfo, storeFile, createBackRefs);
/** * Execute the restore operation * @param snapshotDir The snapshot directory to use as "restore source" * @param sd The snapshot descriptor * @param htdClone The HTableDescriptor of the table to restore/clone. */ private void testRestore(final Path snapshotDir, final SnapshotDescription sd, final TableDescriptor htdClone) throws IOException { LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir); FSUtils.logFileSystemState(fs, rootDir, LOG); new FSTableDescriptors(conf).createTableDescriptor(htdClone); RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone); helper.restoreHdfsRegions(); LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir); FSUtils.logFileSystemState(fs, rootDir, LOG); }
clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
/** * Initialize the restore helper, based on the snapshot and table information provided. */ private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir, final SnapshotDescription sd, final TableDescriptor htdClone) throws IOException { ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class); MonitoredTask status = Mockito.mock(MonitoredTask.class); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd); return new RestoreSnapshotHelper(conf, fs, manifest, htdClone, rootDir, monitor, status); }
/** * Create a new {@link HFileLink} to reference the store file. * <p>The store file in the snapshot can be a simple hfile, an HFileLink or a reference. * <ul> * <li>hfile: abc -> table=region-abc * <li>reference: abc.1234 -> table=region-abc.1234 * <li>hfilelink: table=region-hfile -> table=region-hfile * </ul> * @param familyDir destination directory for the store file * @param regionInfo destination region info for the table * @param hfileName store file name (can be a Reference, HFileLink or simple HFile) */ private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo, final String hfileName) throws IOException { if (HFileLink.isHFileLink(hfileName)) { HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName); } else if (StoreFile.isReference(hfileName)) { restoreReferenceFile(familyDir, regionInfo, hfileName); } else { HFileLink.create(conf, fs, familyDir, regionInfo, hfileName); } }
/** * Clone region directory content from the snapshot info. * * Each region is encoded with the table name, so the cloned region will have * a different region name. * * Instead of copying the hfiles a HFileLink is created. * * @param region {@link HRegion} cloned * @param snapshotRegionInfo */ private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo, final SnapshotRegionManifest manifest) throws IOException { final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName()); final String tableName = tableDesc.getTableName().getNameAsString(); for (SnapshotRegionManifest.FamilyFiles familyFiles: manifest.getFamilyFilesList()) { Path familyDir = new Path(regionDir, familyFiles.getFamilyName().toStringUtf8()); for (SnapshotRegionManifest.StoreFile storeFile: familyFiles.getStoreFilesList()) { LOG.info("Adding HFileLink " + storeFile.getName() + " to table=" + tableName); restoreStoreFile(familyDir, snapshotRegionInfo, storeFile, createBackRefs); } } }
private void openWithRestoringSnapshot() throws IOException { final RestoreSnapshotHelper.RestoreMetaChanges meta = RestoreSnapshotHelper.copySnapshotForScanner(conf, fs, rootDir, restoreDir, snapshotName); final List<RegionInfo> restoredRegions = meta.getRegionsToAdd(); htd = meta.getTableDescriptor(); regions = new ArrayList<>(restoredRegions.size()); restoredRegions.stream().filter(this::isValidRegion).sorted().forEach(r -> regions.add(r)); }
RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper( conf, fs, manifest, tableDescriptor, tableRootDir, monitorException, monitorStatus); RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();