@Test public void testExceptionOnPolicyEnforcementDisable() throws Exception { final TableName tableName = TableName.valueOf("foo"); final SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 2048L); RegionServerServices rss = mock(RegionServerServices.class); SpaceViolationPolicyEnforcementFactory factory = mock( SpaceViolationPolicyEnforcementFactory.class); SpaceViolationPolicyEnforcement enforcement = mock(SpaceViolationPolicyEnforcement.class); RegionServerSpaceQuotaManager realManager = new RegionServerSpaceQuotaManager(rss, factory); when(factory.create(rss, tableName, snapshot)).thenReturn(enforcement); doNothing().when(enforcement).enable(); doThrow(new IOException("Failed for test!")).when(enforcement).disable(); // Enabling should work realManager.enforceViolationPolicy(tableName, snapshot); Map<TableName, SpaceViolationPolicyEnforcement> enforcements = realManager.copyActiveEnforcements(); assertEquals(1, enforcements.size()); // If the disable fails, we should still treat it as "active" realManager.disableViolationPolicyEnforcement(tableName); enforcements = realManager.copyActiveEnforcements(); assertEquals(1, enforcements.size()); } }
getManager().copyQuotaSnapshots(); LOG.trace("Enabling " + newSnapshot + " on " + tableName); getManager().enforceViolationPolicy(tableName, newSnapshot); LOG.trace("Removing quota violation policy on " + tableName); getManager().disableViolationPolicyEnforcement(tableName); LOG.trace("Removing quota violation policy on " + tableName); getManager().disableViolationPolicyEnforcement(tableName); getManager().updateQuotaSnapshot(newSnapshots); } catch (IOException e) { LOG.warn(
final SpaceViolationPolicyEnforcement enforcement = getFactory().create( getRegionServerServices(), tableName, snapshot);
/** * Creates an object well-suited for the RegionServer to use in verifying active policies. */ public ActivePolicyEnforcement getActiveEnforcements() { return new ActivePolicyEnforcement(copyActiveEnforcements(), copyQuotaSnapshots(), rsServices); }
when(manager.copyQuotaSnapshots()).thenReturn(previousPolicies); verify(manager).enforceViolationPolicy( TableName.valueOf("table1"), policiesToEnforce.get(TableName.valueOf("table1"))); verify(manager).enforceViolationPolicy( TableName.valueOf("table2"), policiesToEnforce.get(TableName.valueOf("table2"))); verify(manager).disableViolationPolicyEnforcement(TableName.valueOf("table3")); verify(manager).disableViolationPolicyEnforcement(TableName.valueOf("table4"));
@Test public void testExceptionOnPolicyEnforcementEnable() throws Exception { final TableName tableName = TableName.valueOf("foo"); final SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 2048L); RegionServerServices rss = mock(RegionServerServices.class); SpaceViolationPolicyEnforcementFactory factory = mock( SpaceViolationPolicyEnforcementFactory.class); SpaceViolationPolicyEnforcement enforcement = mock(SpaceViolationPolicyEnforcement.class); RegionServerSpaceQuotaManager realManager = new RegionServerSpaceQuotaManager(rss, factory); when(factory.create(rss, tableName, snapshot)).thenReturn(enforcement); doThrow(new IOException("Failed for test!")).when(enforcement).enable(); realManager.enforceViolationPolicy(tableName, snapshot); Map<TableName, SpaceViolationPolicyEnforcement> enforcements = realManager.copyActiveEnforcements(); assertTrue("Expected active enforcements to be empty, but were " + enforcements, enforcements.isEmpty()); }
@Test public void testNewPolicyOverridesOld() throws IOException { final Map<TableName,SpaceQuotaSnapshot> policiesToEnforce = new HashMap<>(); policiesToEnforce.put( TableName.valueOf("table1"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 512L)); policiesToEnforce.put( TableName.valueOf("table2"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 2048L, 512L)); policiesToEnforce.put( TableName.valueOf("table3"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 512L)); final Map<TableName,SpaceQuotaSnapshot> previousPolicies = new HashMap<>(); previousPolicies.put( TableName.valueOf("table1"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 8192L, 512L)); // No active enforcements when(manager.getActivePoliciesAsMap()).thenReturn(previousPolicies); // Policies to enforce when(chore.fetchSnapshotsFromQuotaTable()).thenReturn(policiesToEnforce); chore.chore(); for (Entry<TableName,SpaceQuotaSnapshot> entry : policiesToEnforce.entrySet()) { verify(manager).enforceViolationPolicy(entry.getKey(), entry.getValue()); } verify(manager, never()).disableViolationPolicyEnforcement(TableName.valueOf("table1")); }
Map<TableName,SpaceQuotaSnapshot> snapshots = spaceQuotaManager.copyQuotaSnapshots(); Map<RegionInfo,Long> regionSizes = getReportedSizesForTable(tn); while (true) { regionSizes); Thread.sleep(3000); snapshots = spaceQuotaManager.copyQuotaSnapshots(); regionSizes = getReportedSizesForTable(tn); ActivePolicyEnforcement activePolicies = spaceQuotaManager.getActiveEnforcements(); SpaceViolationPolicyEnforcement enforcement = activePolicies.getPolicyEnforcement(tn); assertTrue(
@Override public boolean evaluate() throws Exception { SpaceQuotaSnapshot snapshot = manager.copyQuotaSnapshots().get(tn); if (snapshot == null) { return false; } return snapshot.getUsage() >= tableSize; } });
RegionSizeStore getRegionSizeStore() { return rs.getRegionServerSpaceQuotaManager().getRegionSizeStore(); }
final Map<TableName, SpaceViolationPolicyEnforcement> enforcements = new HashMap<>(); final Map<TableName, SpaceQuotaSnapshot> expectedPolicies = new HashMap<>(); when(quotaManager.copyActiveEnforcements()).thenReturn(enforcements); when(quotaManager.getActivePoliciesAsMap()).thenCallRealMethod(); TableName.valueOf("no_policy"), new DefaultViolationPolicyEnforcement()); Map<TableName, SpaceQuotaSnapshot> actualPolicies = quotaManager.getActivePoliciesAsMap(); assertEquals(expectedPolicies, actualPolicies);
ActivePolicyEnforcement activeSpaceQuotas = getSpaceQuotaManager().getActiveEnforcements(); SpaceViolationPolicyEnforcement enforcement = activeSpaceQuotas.getPolicyEnforcement( region); getSpaceQuotaManager().getRegionSizeStore().incrementRegionSize( region.getRegionInfo(), sizeToBeLoaded);
@Override public boolean evaluate() throws Exception { ActivePolicyEnforcement enforcements = manager.getActiveEnforcements(); SpaceViolationPolicyEnforcement enforcement = enforcements.getPolicyEnforcement(tn); // Signifies that we're waiting on the quota snapshot to be fetched if (enforcement instanceof MissingSnapshotViolationPolicyEnforcement) { return false; } return enforcement.getQuotaSnapshot().getQuotaStatus().isInViolation(); } });
public SpaceQuotaRefresherChore(RegionServerSpaceQuotaManager manager, Connection conn) { super(SpaceQuotaRefresherChore.class.getSimpleName(), manager.getRegionServerServices(), getPeriod(manager.getRegionServerServices().getConfiguration()), getInitialDelay(manager.getRegionServerServices().getConfiguration()), getTimeUnit(manager.getRegionServerServices().getConfiguration())); this.manager = manager; this.conn = conn; }
@Test public void testSpaceQuotaViolation() throws IOException, InterruptedException { region.getRegionServerServices().getRegionServerSpaceQuotaManager().enforceViolationPolicy(NAME, new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES_COMPACTIONS), 10L, 100L)); Tracker tracker = new Tracker(); TRACKER = tracker; region.requestCompaction("test", Store.PRIORITY_USER, false, tracker); tracker.await(); assertEquals(2, tracker.notExecutedStores.size()); tracker.notExecutedStores.sort((p1, p2) -> p1.getFirst().getColumnFamilyName() .compareTo(p2.getFirst().getColumnFamilyName())); assertEquals(Bytes.toString(CF1), tracker.notExecutedStores.get(0).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(0).getSecond(), containsString("space quota violation")); assertEquals(Bytes.toString(CF2), tracker.notExecutedStores.get(1).getFirst().getColumnFamilyName()); assertThat(tracker.notExecutedStores.get(1).getSecond(), containsString("space quota violation")); assertTrue(tracker.beforeExecuteStores.isEmpty()); assertTrue(tracker.afterExecuteStores.isEmpty()); } }
/** * Converts a map of table to {@link SpaceViolationPolicyEnforcement}s into * {@link SpaceViolationPolicy}s. */ public Map<TableName, SpaceQuotaSnapshot> getActivePoliciesAsMap() { final Map<TableName, SpaceViolationPolicyEnforcement> enforcements = copyActiveEnforcements(); final Map<TableName, SpaceQuotaSnapshot> policies = new HashMap<>(); for (Entry<TableName, SpaceViolationPolicyEnforcement> entry : enforcements.entrySet()) { final SpaceQuotaSnapshot snapshot = entry.getValue().getQuotaSnapshot(); if (snapshot != null) { policies.put(entry.getKey(), snapshot); } } return policies; }
rsSpaceQuotaManager = new RegionServerSpaceQuotaManager(this);
this.server.getRegionServerSpaceQuotaManager(); if (spaceQuotaManager != null && spaceQuotaManager.areCompactionsDisabled(region.getTableDescriptor().getTableName())) { String reason = "Ignoring compaction request for " + region + " as an active space quota violation " + " policy disallows compactions.";
rsSpaceQuotaManager.buildFileArchiveRequest(tableName, archivedFiles); rss.reportFileArchival(null, request); } catch (ServiceException se) {
@Test public void testPoliciesAreEnforced() throws IOException { // Create a number of policies that should be enforced (usage > limit) final Map<TableName,SpaceQuotaSnapshot> policiesToEnforce = new HashMap<>(); policiesToEnforce.put( TableName.valueOf("table1"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 512L)); policiesToEnforce.put( TableName.valueOf("table2"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 2048L, 512L)); policiesToEnforce.put( TableName.valueOf("table3"), new SpaceQuotaSnapshot(new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES), 4096L, 512L)); policiesToEnforce.put( TableName.valueOf("table4"), new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.NO_WRITES_COMPACTIONS), 8192L, 512L)); // No active enforcements when(manager.copyQuotaSnapshots()).thenReturn(Collections.emptyMap()); // Policies to enforce when(chore.fetchSnapshotsFromQuotaTable()).thenReturn(policiesToEnforce); chore.chore(); for (Entry<TableName,SpaceQuotaSnapshot> entry : policiesToEnforce.entrySet()) { // Ensure we enforce the policy verify(manager).enforceViolationPolicy(entry.getKey(), entry.getValue()); // Don't disable any policies verify(manager, never()).disableViolationPolicyEnforcement(entry.getKey()); } }