@After public void tearDown() throws Exception { SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin()); SnapshotTestingUtils.deleteArchiveDirectory(UTIL); }
@Override public void loadData(final Table table, byte[]... families) throws Exception { SnapshotTestingUtils.loadData(UTIL, table.getName(), 1000, families); }
/** * Take a snapshot of the specified table and verify that the given family is * not empty. Note that this will leave the table disabled * in the case of an offline snapshot. */ public static void createSnapshotAndValidate(Admin admin, TableName tableName, String familyName, String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot) throws Exception { ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<>(1); nonEmptyFamilyNames.add(Bytes.toBytes(familyName)); createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null, snapshotNameString, rootDir, fs, onlineSnapshot); }
/** * Take a snapshot of the specified table and verify the given families. * Note that this will leave the table disabled in the case of an offline snapshot. */ public static void createSnapshotAndValidate(Admin admin, TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames, String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot) throws Exception { if (!onlineSnapshot) { try { LOG.info("prepping for offline snapshot."); admin.disableTable(tableName); } catch (TableNotEnabledException tne) { LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " + "already disabled."); } } LOG.info("taking snapshot."); admin.snapshot(snapshotNameString, tableName); LOG.info("Confirming snapshot exists."); List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName); if (snapshots == null || snapshots.size() != 1) { Assert.fail("Incorrect number of snapshots for table " + tableName); } LOG.info("validating snapshot."); SnapshotTestingUtils.confirmSnapshotValid( ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames, emptyFamilyNames, rootDir, admin, fs); }
private SnapshotProtos.SnapshotDescription getSnapshot() throws Exception { if (snapshot == null) { final TableName snapshotTableName = TableName.valueOf("testCloneSnapshot"); long tid = System.currentTimeMillis(); final byte[] snapshotName = Bytes.toBytes("snapshot-" + tid); Admin admin = UTIL.getAdmin(); // create Table SnapshotTestingUtils.createTable(UTIL, snapshotTableName, getNumReplicas(), CF); // Load data SnapshotTestingUtils.loadData(UTIL, snapshotTableName, 500, CF); admin.disableTable(snapshotTableName); // take a snapshot admin.snapshot(snapshotName, snapshotTableName); admin.enableTable(snapshotTableName); List<SnapshotDescription> snapshotList = admin.listSnapshots(); snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotList.get(0)); } return snapshot; }
/** * Test simple flush snapshotting a table that is online * @throws Exception */ @Test public void testFlushTableSnapshot() throws Exception { // make sure we don't fail on listing snapshots SnapshotTestingUtils.assertNoSnapshots(admin); // put some stuff in the table SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); LOG.debug("FS state before snapshot:"); UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); // take a snapshot of the enabled table String snapshotString = "offlineTableSnapshot"; byte[] snapshot = Bytes.toBytes(snapshotString); admin.snapshot(snapshotString, TABLE_NAME, SnapshotType.FLUSH); LOG.debug("Snapshot completed."); // make sure we have the snapshot List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME); // make sure its a valid snapshot LOG.debug("FS state after snapshot:"); UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); SnapshotTestingUtils.confirmSnapshotValid(UTIL, ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM); }
/** * Basic end-to-end test of simple-flush-based snapshots */ @Test public void testFlushCreateListDestroy() throws Exception { LOG.debug("------- Starting Snapshot test -------------"); // make sure we don't fail on listing snapshots SnapshotTestingUtils.assertNoSnapshots(admin); // load the table so we have some data SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); String snapshotName = "flushSnapshotCreateListDestroy"; FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM), snapshotName, rootDir, fs, true); }
int numRows = DEFAULT_NUM_ROWS; SnapshotTestingUtils.assertNoSnapshots(admin); SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM); SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME, SnapshotType.FLUSH, 3); SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName);
int numRows = DEFAULT_NUM_ROWS; SnapshotTestingUtils.assertNoSnapshots(admin); SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM); SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneBeforeMergeName); SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneAfterMergeName);
SnapshotTestingUtils.createTable(UTIL, tableName, getNumReplicas(), TEST_FAM); SnapshotTestingUtils.loadData(UTIL, tableName, 500, TEST_FAM); try (Table table = UTIL.getConnection().getTable(tableName)) { snapshot0Rows = UTIL.countRows(table); SnapshotTestingUtils.loadData(UTIL, tableName, 500, TEST_FAM); try (Table table = UTIL.getConnection().getTable(tableName)) { snapshot1Rows = UTIL.countRows(table); SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows); takeSnapshot(tableName, Bytes.toString(snapshotName1), false); admin.restoreSnapshot(snapshotName0); admin.enableTable(tableName); SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot0Rows); SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas()); admin.restoreSnapshot(emptySnapshot); admin.enableTable(tableName); SnapshotTestingUtils.verifyRowCount(UTIL, tableName, 0); SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas()); admin.restoreSnapshot(snapshotName1); admin.enableTable(tableName); SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows); SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas()); SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows);
Admin admin = UTIL.getAdmin(); SnapshotTestingUtils.createTable(UTIL, snapshotTableName, getNumReplicas(), CF1, CF2); SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF1, CF1); SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF2, CF2); SnapshotTestingUtils.verifyRowCount(UTIL, snapshotTableName, rowCountCF1 + rowCountCF2); SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF3, CF3); SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF4, CF4); SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF1addition, CF1); HTableDescriptor currentHTD = admin.getTableDescriptor(snapshotTableName); assertTrue(currentHTD.hasFamily(CF1)); assertTrue(currentHTD.hasFamily(CF4)); assertNotEquals(currentHTD.getFamiliesKeys().size(), snapshotHTD.getFamiliesKeys().size()); SnapshotTestingUtils.verifyRowCount( UTIL, snapshotTableName, rowCountCF1 + rowCountCF3 + rowCountCF4 + rowCountCF1addition); admin.disableTable(snapshotTableName);
SnapshotTestingUtils.assertNoSnapshots(admin); SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM); SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
@Test public void testRestoreSnapshotAfterTruncate() throws Exception { TableName tableName = TableName.valueOf(getValidMethodName()); SnapshotTestingUtils.createTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY); SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY); int numOfRows = 0; try (Table table = TEST_UTIL.getConnection().getTable(tableName)) { numOfRows = countRows(table); } // take snapshot admin.snapshot("snap", tableName); admin.disableTable(tableName); admin.truncateTable(tableName, false); admin.disableTable(tableName); admin.restoreSnapshot("snap"); admin.enableTable(tableName); verifyRowCount(TEST_UTIL, tableName, numOfRows); SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas()); } }
@After public void tearDown() throws Exception { SnapshotTestingUtils.deleteAllSnapshots(admin); }
private void waitRegionsAfterMerge(final long numRegionsAfterMerge) throws IOException, InterruptedException { // Verify that there's one region less long startTime = System.currentTimeMillis(); while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) { // This may be flaky... if after 15sec the merge is not complete give up // it will fail in the assertEquals(numRegionsAfterMerge). if ((System.currentTimeMillis() - startTime) > 15000) break; Thread.sleep(100); } SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME); }
public static void loadData(final HBaseTestingUtility util, final BufferedMutator mutator, int rows, byte[]... families) throws IOException, InterruptedException { // Ensure one row per region assertTrue(rows >= KEYS.length); for (byte k0: KEYS) { byte[] k = new byte[] { k0 }; byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k); byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value))); final byte[][] families1 = families; final byte[] key1 = key; final byte[] value1 = value; mutator.mutate(createPut(families1, key1, value1)); rows--; } // Add other extra rows. more rows, more files while (rows-- > 0) { byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows)); byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value)); final byte[][] families1 = families; final byte[] key1 = key; final byte[] value1 = value; mutator.mutate(createPut(families1, key1, value1)); } mutator.flush(); waitForTableToBeOnline(util, mutator.getName()); }
protected void createTable() throws Exception { SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY); }
/** * Test simple flush snapshotting a table that is online * @throws Exception */ @Test public void testFlushTableSnapshotWithProcedure() throws Exception { // make sure we don't fail on listing snapshots SnapshotTestingUtils.assertNoSnapshots(admin); // put some stuff in the table SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); LOG.debug("FS state before snapshot:"); UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); // take a snapshot of the enabled table String snapshotString = "offlineTableSnapshot"; byte[] snapshot = Bytes.toBytes(snapshotString); Map<String, String> props = new HashMap<>(); props.put("table", TABLE_NAME.getNameAsString()); admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, snapshotString, props); LOG.debug("Snapshot completed."); // make sure we have the snapshot List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME); // make sure its a valid snapshot LOG.debug("FS state after snapshot:"); UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG); SnapshotTestingUtils.confirmSnapshotValid(UTIL, ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM); }
/** * Basic end-to-end test of simple-flush-based snapshots */ @Test public void testFlushCreateListDestroy() throws Exception { LOG.debug("------- Starting Snapshot test -------------"); // make sure we don't fail on listing snapshots SnapshotTestingUtils.assertNoSnapshots(admin); // load the table so we have some data SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM); String snapshotName = "flushSnapshotCreateListDestroy"; FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem(); Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir(); SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM), snapshotName, rootDir, fs, true); }
int numRows = DEFAULT_NUM_ROWS; SnapshotTestingUtils.assertNoSnapshots(admin); SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM); SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME, SnapshotType.FLUSH, 3); SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName);