TableName createTable() throws Exception { return createTableWithRegions(1); }
TableName createTableWithRegions(int numRegions) throws Exception { return createTableWithRegions(NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR, numRegions); }
TableName createTableWithRegions(Admin admin, int numRegions) throws Exception { return createTableWithRegions( testUtil.getAdmin(), NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR, numRegions); }
TableName createTableWithRegions(String namespace, int numRegions) throws Exception { return createTableWithRegions(testUtil.getAdmin(), namespace, numRegions); }
private TableName writeUntilViolation(SpaceViolationPolicy policyToViolate) throws Exception { TableName tn = helper.createTableWithRegions(10); setQuotaLimit(tn, policyToViolate, 2L); // Write more data than should be allowed and flush it to disk helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE); // This should be sufficient time for the chores to run and see the change. Thread.sleep(5000); return tn; }
TableName tn1 = helper.createTableWithRegions(20); TableName tn2 = helper.createTableWithRegions(20); TableName tn3 = helper.createTableWithRegions(20);
@Override public TableName call() throws Exception { try (Connection conn = getConnection()) { final Admin admin = conn.getAdmin(); final TableName tn = helper.createTableWithRegions(admin, 5); final long sizeLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, sizeLimit, SpaceViolationPolicy.NO_WRITES_COMPACTIONS); admin.setQuota(settings); // Grant the normal user permission to create a table and set a quota try { AccessControlClient.grant( conn, tn, REGULARUSER_NAME, null, null, Action.READ, Action.WRITE); } catch (Throwable t) { if (t instanceof Exception) { throw (Exception) t; } throw new Exception(t); } return tn; } } });
@Override public TableName call() throws Exception { try (Connection conn = getConnection()) { Admin admin = conn.getAdmin(); final TableName tn = helper.createTableWithRegions(admin, 5); final long sizeLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, sizeLimit, SpaceViolationPolicy.NO_WRITES_COMPACTIONS); admin.setQuota(settings); // Grant the normal user permissions try { AccessControlClient.grant( conn, tn, REGULARUSER_NAME, null, null, Action.READ, Action.WRITE); } catch (Throwable t) { if (t instanceof Exception) { throw (Exception) t; } throw new Exception(t); } return tn; } } });
@Test public void testTableQuotaOverridesNamespaceQuota() throws Exception { final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS; final TableName tn = helper.createTableWithRegions(10); // 2MB limit on the table, 1GB limit on the namespace final long tableLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; final long namespaceLimit = 1024L * SpaceQuotaHelperForTests.ONE_MEGABYTE; TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy)); TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitNamespaceSpace( tn.getNamespaceAsString(), namespaceLimit, policy)); // Write more data than should be allowed and flush it to disk helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE); // This should be sufficient time for the chores to run and see the change. Thread.sleep(5000); // The write should be rejected because the table quota takes priority over the namespace Put p = new Put(Bytes.toBytes("to_reject")); p.addColumn( Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject")); verifyViolation(policy, tn, p); }
@Test public void testBulkLoading() throws Exception { TableName tn = helper.createTableWithRegions(1); // Set a quota QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); admin.setQuota(settings); ClientServiceCallable<Boolean> callable = helper.generateFileToLoad(tn, 3, 550); // Make sure the files are about as long as we expect FileSystem fs = TEST_UTIL.getTestFileSystem(); FileStatus[] files = fs.listStatus( new Path(fs.getHomeDirectory(), testName.getMethodName() + "_files")); long totalSize = 0; for (FileStatus file : files) { assertTrue( "Expected the file, " + file.getPath() + ", length to be larger than 25KB, but was " + file.getLen(), file.getLen() > 25 * SpaceQuotaHelperForTests.ONE_KILOBYTE); totalSize += file.getLen(); } RpcRetryingCallerFactory factory = new RpcRetryingCallerFactory(TEST_UTIL.getConfiguration()); RpcRetryingCaller<Boolean> caller = factory.<Boolean> newCaller(); assertTrue("The bulk load failed", caller.callWithRetries(callable, Integer.MAX_VALUE)); final long finalTotalSize = totalSize; TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { return snapshot.getUsage() >= finalTotalSize; } }); }
@Test public void testSnapshotsFromTables() throws Exception { TableName tn1 = helper.createTableWithRegions(1); TableName tn2 = helper.createTableWithRegions(1); TableName tn3 = helper.createTableWithRegions(1); // Set a space quota on table 1 and 2 (but not 3) admin.setQuota(QuotaSettingsFactory.limitTableSpace( tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS)); admin.setQuota(QuotaSettingsFactory.limitTableSpace( tn2, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS)); // Create snapshots on each table (we didn't write any data, so just skipflush) admin.snapshot(new SnapshotDescription(tn1 + "snapshot", tn1, SnapshotType.SKIPFLUSH)); admin.snapshot(new SnapshotDescription(tn2 + "snapshot", tn2, SnapshotType.SKIPFLUSH)); admin.snapshot(new SnapshotDescription(tn3 + "snapshot", tn3, SnapshotType.SKIPFLUSH)); Multimap<TableName,String> mapping = testChore.getSnapshotsToComputeSize(); assertEquals(2, mapping.size()); assertEquals(1, mapping.get(tn1).size()); assertEquals(tn1 + "snapshot", mapping.get(tn1).iterator().next()); assertEquals(1, mapping.get(tn2).size()); assertEquals(tn2 + "snapshot", mapping.get(tn2).iterator().next()); admin.snapshot(new SnapshotDescription(tn2 + "snapshot1", tn2, SnapshotType.SKIPFLUSH)); admin.snapshot(new SnapshotDescription(tn3 + "snapshot1", tn3, SnapshotType.SKIPFLUSH)); mapping = testChore.getSnapshotsToComputeSize(); assertEquals(3, mapping.size()); assertEquals(1, mapping.get(tn1).size()); assertEquals(tn1 + "snapshot", mapping.get(tn1).iterator().next()); assertEquals(2, mapping.get(tn2).size()); assertEquals( new HashSet<String>(Arrays.asList(tn2 + "snapshot", tn2 + "snapshot1")), mapping.get(tn2)); }
@Test public void testFlushes() throws Exception { TableName tn = helper.createTableWithRegions(1); // Set a quota QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); admin.setQuota(settings); // Write some data final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; helper.writeData(tn, initialSize); // Make sure a flush happened admin.flush(tn); // We should be able to observe the system recording an increase in size (even // though we know the filesystem scanning did not happen). TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { return snapshot.getUsage() >= initialSize; } }); }
@Test public void testRegionSizesFromMaster() throws Exception { final long tableSize = 1024L * 10L; // 10KB final int numRegions = 10; final TableName tn = helper.createTableWithRegions(numRegions); // Will write at least `tableSize` data helper.writeData(tn, tableSize); final HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); final MasterQuotaManager quotaManager = master.getMasterQuotaManager(); // Make sure the master has all of the reports Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() { @Override public boolean evaluate() throws Exception { Map<RegionInfo,Long> regionSizes = quotaManager.snapshotRegionSizes(); LOG.trace("Region sizes=" + regionSizes); return numRegions == countRegionsForTable(tn, regionSizes) && tableSize <= getTableSize(tn, regionSizes); } }); Map<TableName, Long> sizes = TEST_UTIL.getAdmin().getSpaceQuotaTableSizes(); Long size = sizes.get(tn); assertNotNull("No reported size for " + tn, size); assertTrue("Reported table size was " + size, size.longValue() >= tableSize); }
@Test public void testTableViolatesQuota() throws Exception { TableName tn = helper.createTableWithRegions(10);
admin.createNamespace(ns); TableName tn1 = helper.createTableWithRegions(ns.getName(), 1); TableName tn2 = helper.createTableWithRegions(ns.getName(), 1); TableName tn3 = helper.createTableWithRegions(1);
@Test public void testMajorCompaction() throws Exception { TableName tn = helper.createTableWithRegions(1); // Set a quota QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS); admin.setQuota(settings); // Write some data and flush it to disk. final long sizePerBatch = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE; helper.writeData(tn, sizePerBatch); admin.flush(tn); // Write the same data again, flushing it to a second file helper.writeData(tn, sizePerBatch); admin.flush(tn); // After two flushes, both hfiles would contain similar data. We should see 2x the data. TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { return snapshot.getUsage() >= 2L * sizePerBatch; } }); // Rewrite the two files into one. admin.majorCompact(tn); // After we major compact the table, we should notice quickly that the amount of data in the // table is much closer to reality (the duplicate entries across the two files are removed). TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) { @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception { return snapshot.getUsage() >= sizePerBatch && snapshot.getUsage() <= 2L * sizePerBatch; } }); }
final long tableSize = 1024L * 10L; // 10KB final int numRegions = 10; final TableName tn = helper.createTableWithRegions(numRegions);
@Test public void testMinorCompaction() throws Exception { TableName tn = helper.createTableWithRegions(1);
final long tableSize = 1024L * 10L; // 10KB final int numRegions = 10; final TableName tn = helper.createTableWithRegions(numRegions);
final TableName tn1 = helper.createTableWithRegions(1); admin.setQuota(QuotaSettingsFactory.limitTableSpace( tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));