/** * Check that ExportSnapshot will succeed if something fails but the retry succeed. */ @Test public void testExportRetry() throws Exception { Path copyDir = getLocalDestinationDir(); FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); copyDir = copyDir.makeQualified(fs); Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true); conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 2); conf.setInt("mapreduce.map.maxattempts", 3); testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true, getBypassRegionPredicate(), true); }
private void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, final byte[] targetName, int filesExpected) throws Exception { testExportFileSystemState(tableName, snapshotName, targetName, filesExpected, getHdfsDestinationDir(), false); }
@Test public void testConsecutiveExports() throws Exception { Path copyDir = getLocalDestinationDir(); testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false); testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true); removeExportDir(copyDir); }
protected Set<String> verifySnapshot(final FileSystem fs, final Path rootDir, final TableName tableName, final String snapshotName) throws IOException { return verifySnapshot(TEST_UTIL.getConfiguration(), fs, rootDir, tableName, snapshotName, getBypassRegionPredicate()); }
/** * Verify if exported snapshot and copied files matches the original one. */ @Test public void testExportFileSystemState() throws Exception { testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles); }
protected void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite) throws Exception { testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName, filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite, getBypassRegionPredicate(), true); }
final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName)); final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName)); verifySnapshotDir(hdfs, new Path(sourceDir, snapshotDir), fs, new Path(copyDir, targetDir)); Set<String> snapshotFiles = verifySnapshot(conf, fs, copyDir, tableName, Bytes.toString(targetName), bypassregionPredicate); assertEquals(filesExpected, snapshotFiles.size());
public static void setUpBaseConf(Configuration conf) { TestExportSnapshot.setUpBaseConf(conf); conf.setInt(MobConstants.MOB_FILE_CACHE_SIZE_KEY, 0); }
/** * Create a table and take a snapshot of the table used by the export test. */ @Before public void setUp() throws Exception { this.admin = TEST_UTIL.getAdmin(); tableName = TableName.valueOf("testtb-" + testName.getMethodName()); snapshotName = Bytes.toBytes("snaptb0-" + testName.getMethodName()); emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + testName.getMethodName()); // create Table createTable(); // Take an empty snapshot admin.snapshot(emptySnapshotName, tableName); // Add some rows SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); tableNumFiles = admin.getTableRegions(tableName).size(); // take a snapshot admin.snapshot(snapshotName, tableName); }
@Test public void testConsecutiveExports() throws Exception { Path copyDir = getLocalDestinationDir(); testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false); testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true); removeExportDir(copyDir); }
@Test public void testExportWithTargetName() throws Exception { final byte[] targetName = Bytes.toBytes("testExportWithTargetName"); testExportFileSystemState(tableName, snapshotName, targetName, tableNumFiles); }
protected Set<String> verifySnapshot(final FileSystem fs, final Path rootDir, final TableName tableName, final String snapshotName) throws IOException { return verifySnapshot(TEST_UTIL.getConfiguration(), fs, rootDir, tableName, snapshotName, getBypassRegionPredicate()); }
protected void testExportFileSystemState(final TableName tableName, final byte[] snapshotName, final byte[] targetName, int filesExpected, Path copyDir, boolean overwrite) throws Exception { testExportFileSystemState(TEST_UTIL.getConfiguration(), tableName, snapshotName, targetName, filesExpected, TEST_UTIL.getDefaultRootDirPath(), copyDir, overwrite, getBypassRegionPredicate(), true); }
final Path snapshotDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(snapshotName)); final Path targetDir = new Path(HConstants.SNAPSHOT_DIR_NAME, Bytes.toString(targetName)); verifySnapshotDir(hdfs, new Path(sourceDir, snapshotDir), fs, new Path(copyDir, targetDir)); Set<String> snapshotFiles = verifySnapshot(conf, fs, copyDir, tableName, Bytes.toString(targetName), bypassregionPredicate); assertEquals(filesExpected, snapshotFiles.size());
public static void setUpBaseConf(Configuration conf) { TestExportSnapshot.setUpBaseConf(conf); conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR, "file://" + TEMP_DIR + "/.tmpdir/"); } }
/** * Create a table and take a snapshot of the table used by the export test. */ @Before public void setUp() throws Exception { this.admin = TEST_UTIL.getAdmin(); tableName = TableName.valueOf("testtb-" + testName.getMethodName()); snapshotName = Bytes.toBytes("snaptb0-" + testName.getMethodName()); emptySnapshotName = Bytes.toBytes("emptySnaptb0-" + testName.getMethodName()); // create Table createTable(); // Take an empty snapshot admin.snapshot(emptySnapshotName, tableName); // Add some rows SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 50, FAMILY); tableNumFiles = admin.getTableRegions(tableName).size(); // take a snapshot admin.snapshot(snapshotName, tableName); }
/** * Check that ExportSnapshot will fail if we inject failure more times than MR will retry. */ @Test public void testExportFailure() throws Exception { Path copyDir = getLocalDestinationDir(); FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration()); copyDir = copyDir.makeQualified(fs); Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); conf.setBoolean(ExportSnapshot.Testing.CONF_TEST_FAILURE, true); conf.setInt(ExportSnapshot.Testing.CONF_TEST_FAILURE_COUNT, 4); conf.setInt("mapreduce.map.maxattempts", 3); testExportFileSystemState(conf, tableName, snapshotName, snapshotName, tableNumFiles, TEST_UTIL.getDefaultRootDirPath(), copyDir, true, getBypassRegionPredicate(), false); }
@Test public void testConsecutiveExports() throws Exception { Path copyDir = getLocalDestinationDir(); testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, false); testExportFileSystemState(tableName, snapshotName, snapshotName, tableNumFiles, copyDir, true); removeExportDir(copyDir); }
@Test public void testEmptyExportFileSystemState() throws Exception { testExportFileSystemState(tableName, emptySnapshotName, emptySnapshotName, 0); }