How to use

SnapshotTestingUtils

in

org.apache.hadoop.hbase.snapshot

@After
public void tearDown() throws Exception {
 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin());
 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
}

@Override
public void loadData(final Table table, byte[]... families) throws Exception {
 SnapshotTestingUtils.loadData(UTIL, table.getName(), 1000, families);
}

/**
 * Take a snapshot of the specified table and verify that the given family is
 * not empty. Note that this will leave the table disabled
 * in the case of an offline snapshot.
 */
public static void createSnapshotAndValidate(Admin admin,
  TableName tableName, String familyName, String snapshotNameString,
  Path rootDir, FileSystem fs, boolean onlineSnapshot)
  throws Exception {
 ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<>(1);
 nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
 createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
              snapshotNameString, rootDir, fs, onlineSnapshot);
}

/**
 * Take a snapshot of the specified table and verify the given families.
 * Note that this will leave the table disabled in the case of an offline snapshot.
 */
public static void createSnapshotAndValidate(Admin admin,
  TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
  String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
   throws Exception {
 if (!onlineSnapshot) {
  try {
   LOG.info("prepping for offline snapshot.");
   admin.disableTable(tableName);
  } catch (TableNotEnabledException tne) {
   LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
     "already disabled.");
  }
 }
 LOG.info("taking snapshot.");
 admin.snapshot(snapshotNameString, tableName);
 LOG.info("Confirming snapshot exists.");
 List<SnapshotDescription> snapshots =
   SnapshotTestingUtils.assertExistsMatchingSnapshot(admin, snapshotNameString, tableName);
 if (snapshots == null || snapshots.size() != 1) {
  Assert.fail("Incorrect number of snapshots for table " + tableName);
 }
 LOG.info("validating snapshot.");
 SnapshotTestingUtils.confirmSnapshotValid(
  ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), tableName, nonEmptyFamilyNames,
  emptyFamilyNames, rootDir, admin, fs);
}

private SnapshotProtos.SnapshotDescription getSnapshot() throws Exception {
 if (snapshot == null) {
  final TableName snapshotTableName = TableName.valueOf("testCloneSnapshot");
  long tid = System.currentTimeMillis();
  final byte[] snapshotName = Bytes.toBytes("snapshot-" + tid);
  Admin admin = UTIL.getAdmin();
  // create Table
  SnapshotTestingUtils.createTable(UTIL, snapshotTableName, getNumReplicas(), CF);
  // Load data
  SnapshotTestingUtils.loadData(UTIL, snapshotTableName, 500, CF);
  admin.disableTable(snapshotTableName);
  // take a snapshot
  admin.snapshot(snapshotName, snapshotTableName);
  admin.enableTable(snapshotTableName);
  List<SnapshotDescription> snapshotList = admin.listSnapshots();
  snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshotList.get(0));
 }
 return snapshot;
}

/**
 * Test simple flush snapshotting a table that is online
 * @throws Exception
 */
@Test
public void testFlushTableSnapshot() throws Exception {
 // make sure we don't fail on listing snapshots
 SnapshotTestingUtils.assertNoSnapshots(admin);
 // put some stuff in the table
 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
 LOG.debug("FS state before snapshot:");
 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
 // take a snapshot of the enabled table
 String snapshotString = "offlineTableSnapshot";
 byte[] snapshot = Bytes.toBytes(snapshotString);
 admin.snapshot(snapshotString, TABLE_NAME, SnapshotType.FLUSH);
 LOG.debug("Snapshot completed.");
 // make sure we have the snapshot
 List<SnapshotDescription> snapshots =
   SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
 // make sure its a valid snapshot
 LOG.debug("FS state after snapshot:");
 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
 SnapshotTestingUtils.confirmSnapshotValid(UTIL,
  ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);
}

/**
 * Basic end-to-end test of simple-flush-based snapshots
 */
@Test
public void testFlushCreateListDestroy() throws Exception {
 LOG.debug("------- Starting Snapshot test -------------");
 // make sure we don't fail on listing snapshots
 SnapshotTestingUtils.assertNoSnapshots(admin);
 // load the table so we have some data
 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
 String snapshotName = "flushSnapshotCreateListDestroy";
 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
 SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM),
  snapshotName, rootDir, fs, true);
}

int numRows = DEFAULT_NUM_ROWS;
SnapshotTestingUtils.assertNoSnapshots(admin);
SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME, SnapshotType.FLUSH, 3);
SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName);

int numRows = DEFAULT_NUM_ROWS;
SnapshotTestingUtils.assertNoSnapshots(admin);
SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneBeforeMergeName);
SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneAfterMergeName);

SnapshotTestingUtils.createTable(UTIL, tableName, getNumReplicas(), TEST_FAM);
SnapshotTestingUtils.loadData(UTIL, tableName, 500, TEST_FAM);
try (Table table = UTIL.getConnection().getTable(tableName)) {
 snapshot0Rows = UTIL.countRows(table);
SnapshotTestingUtils.loadData(UTIL, tableName, 500, TEST_FAM);
try (Table table = UTIL.getConnection().getTable(tableName)) {
 snapshot1Rows = UTIL.countRows(table);
SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows);
takeSnapshot(tableName, Bytes.toString(snapshotName1), false);
admin.restoreSnapshot(snapshotName0);
admin.enableTable(tableName);
SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot0Rows);
SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
admin.restoreSnapshot(emptySnapshot);
admin.enableTable(tableName);
SnapshotTestingUtils.verifyRowCount(UTIL, tableName, 0);
SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
admin.restoreSnapshot(snapshotName1);
admin.enableTable(tableName);
SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows);
SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows);

Admin admin = UTIL.getAdmin();
SnapshotTestingUtils.createTable(UTIL, snapshotTableName, getNumReplicas(), CF1, CF2);
SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF1, CF1);
SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF2, CF2);
SnapshotTestingUtils.verifyRowCount(UTIL, snapshotTableName, rowCountCF1 + rowCountCF2);
SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF3, CF3);
SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF4, CF4);
SnapshotTestingUtils.loadData(UTIL, snapshotTableName, rowCountCF1addition, CF1);
HTableDescriptor currentHTD = admin.getTableDescriptor(snapshotTableName);
assertTrue(currentHTD.hasFamily(CF1));
assertTrue(currentHTD.hasFamily(CF4));
assertNotEquals(currentHTD.getFamiliesKeys().size(), snapshotHTD.getFamiliesKeys().size());
SnapshotTestingUtils.verifyRowCount(
 UTIL, snapshotTableName, rowCountCF1 + rowCountCF3 + rowCountCF4 + rowCountCF1addition);
admin.disableTable(snapshotTableName);

SnapshotTestingUtils.assertNoSnapshots(admin);
SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);

 @Test
 public void testRestoreSnapshotAfterTruncate() throws Exception {
  TableName tableName = TableName.valueOf(getValidMethodName());
  SnapshotTestingUtils.createTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY);
  SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
  int numOfRows = 0;

  try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
   numOfRows = countRows(table);
  }
  // take snapshot
  admin.snapshot("snap", tableName);
  admin.disableTable(tableName);
  admin.truncateTable(tableName, false);
  admin.disableTable(tableName);
  admin.restoreSnapshot("snap");

  admin.enableTable(tableName);
  verifyRowCount(TEST_UTIL, tableName, numOfRows);
  SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
 }
}

@After
public void tearDown() throws Exception {
 SnapshotTestingUtils.deleteAllSnapshots(admin);
}

private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
  throws IOException, InterruptedException {
 // Verify that there's one region less
 long startTime = System.currentTimeMillis();
 while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
  // This may be flaky... if after 15sec the merge is not complete give up
  // it will fail in the assertEquals(numRegionsAfterMerge).
  if ((System.currentTimeMillis() - startTime) > 15000)
   break;
  Thread.sleep(100);
 }
 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
}

public static void loadData(final HBaseTestingUtility util, final BufferedMutator mutator, int rows,
  byte[]... families) throws IOException, InterruptedException {
 // Ensure one row per region
 assertTrue(rows >= KEYS.length);
 for (byte k0: KEYS) {
  byte[] k = new byte[] { k0 };
  byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
  byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
  final byte[][] families1 = families;
  final byte[] key1 = key;
  final byte[] value1 = value;
  mutator.mutate(createPut(families1, key1, value1));
  rows--;
 }
 // Add other extra rows. more rows, more files
 while (rows-- > 0) {
  byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
  byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
  final byte[][] families1 = families;
  final byte[] key1 = key;
  final byte[] value1 = value;
  mutator.mutate(createPut(families1, key1, value1));
 }
 mutator.flush();
 waitForTableToBeOnline(util, mutator.getName());
}

protected void createTable() throws Exception {
 SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY);
}

/**
 * Test simple flush snapshotting a table that is online
 * @throws Exception
 */
@Test
public void testFlushTableSnapshotWithProcedure() throws Exception {
 // make sure we don't fail on listing snapshots
 SnapshotTestingUtils.assertNoSnapshots(admin);
 // put some stuff in the table
 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
 LOG.debug("FS state before snapshot:");
 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
 // take a snapshot of the enabled table
 String snapshotString = "offlineTableSnapshot";
 byte[] snapshot = Bytes.toBytes(snapshotString);
 Map<String, String> props = new HashMap<>();
 props.put("table", TABLE_NAME.getNameAsString());
 admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
   snapshotString, props);
 LOG.debug("Snapshot completed.");
 // make sure we have the snapshot
 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
  snapshot, TABLE_NAME);
 // make sure its a valid snapshot
 LOG.debug("FS state after snapshot:");
 UTIL.getHBaseCluster().getMaster().getMasterFileSystem().logFileSystemState(LOG);
 SnapshotTestingUtils.confirmSnapshotValid(UTIL,
  ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM);
}

/**
 * Basic end-to-end test of simple-flush-based snapshots
 */
@Test
public void testFlushCreateListDestroy() throws Exception {
 LOG.debug("------- Starting Snapshot test -------------");
 // make sure we don't fail on listing snapshots
 SnapshotTestingUtils.assertNoSnapshots(admin);
 // load the table so we have some data
 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
 String snapshotName = "flushSnapshotCreateListDestroy";
 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
 SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM),
  snapshotName, rootDir, fs, true);
}

int numRows = DEFAULT_NUM_ROWS;
SnapshotTestingUtils.assertNoSnapshots(admin);
SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME, SnapshotType.FLUSH, 3);
SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName);