@Test(expected = IllegalArgumentException.class) public void testFileIsNotAFile() throws Exception { final List<String> paths = new ArrayList<>(); String path = "/1"; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(1000L); when(status.isFile()).thenReturn(false); paths.add(path); // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, Long.MAX_VALUE); policy.initialize(rss, tableName, snapshot); // If the file to bulk load isn't a file, this should throw an exception policy.computeBulkLoadSize(fs, paths); }
/** * Disables enforcement on any violation policy on the given <code>tableName</code>. */ public void disableViolationPolicyEnforcement(TableName tableName) { if (LOG.isTraceEnabled()) { LOG.trace("Disabling violation policy enforcement on " + tableName); } // "Disables" the policy synchronized (enforcedPolicies) { SpaceViolationPolicyEnforcement enforcement = enforcedPolicies.remove(tableName); if (enforcement != null) { try { enforcement.disable(); } catch (IOException e) { LOG.error("Failed to disable space violation policy for " + tableName + ". This table will remain in violation.", e); enforcedPolicies.put(tableName, enforcement); } } } }
enforcement.enable(); } catch (IOException e) { LOG.error("Failed to enable space violation policy for " + tableName
@Test(expected = IllegalArgumentException.class) public void testFileIsNotAFile() throws Exception { final List<String> paths = new ArrayList<>(); String path = "/1"; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(1000L); when(status.isFile()).thenReturn(false); paths.add(path); // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, Long.MAX_VALUE); policy.initialize(rss, tableName, snapshot); // If the file to bulk load isn't a file, this should throw an exception policy.checkBulkLoad(fs, paths); }
@Test public void testExceptionOnPolicyEnforcementDisable() throws Exception { final TableName tableName = TableName.valueOf("foo"); final SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 2048L); RegionServerServices rss = mock(RegionServerServices.class); SpaceViolationPolicyEnforcementFactory factory = mock( SpaceViolationPolicyEnforcementFactory.class); SpaceViolationPolicyEnforcement enforcement = mock(SpaceViolationPolicyEnforcement.class); RegionServerSpaceQuotaManager realManager = new RegionServerSpaceQuotaManager(rss, factory); when(factory.create(rss, tableName, snapshot)).thenReturn(enforcement); doNothing().when(enforcement).enable(); doThrow(new IOException("Failed for test!")).when(enforcement).disable(); // Enabling should work realManager.enforceViolationPolicy(tableName, snapshot); Map<TableName, SpaceViolationPolicyEnforcement> enforcements = realManager.copyActiveEnforcements(); assertEquals(1, enforcements.size()); // If the disable fails, we should still treat it as "active" realManager.disableViolationPolicyEnforcement(tableName); enforcements = realManager.copyActiveEnforcements(); assertEquals(1, enforcements.size()); } }
/** * Converts a map of table to {@link SpaceViolationPolicyEnforcement}s into * {@link SpaceViolationPolicy}s. */ public Map<TableName, SpaceQuotaSnapshot> getActivePoliciesAsMap() { final Map<TableName, SpaceViolationPolicyEnforcement> enforcements = copyActiveEnforcements(); final Map<TableName, SpaceQuotaSnapshot> policies = new HashMap<>(); for (Entry<TableName, SpaceViolationPolicyEnforcement> entry : enforcements.entrySet()) { final SpaceQuotaSnapshot snapshot = entry.getValue().getQuotaSnapshot(); if (snapshot != null) { policies.put(entry.getKey(), snapshot); } } return policies; }
++countOfCompleteMutation; checkCellSizeLimit(region, put); spaceQuotaEnforcement.getPolicyEnforcement(region).check(put); rm.add(put); break; Delete del = ProtobufUtil.toDelete(action.getMutation(), cellScanner); ++countOfCompleteMutation; spaceQuotaEnforcement.getPolicyEnforcement(region).check(del); rm.add(del); break;
/** * Constructs the appropriate {@link SpaceViolationPolicyEnforcement} for tables that are * in violation of their space quota. */ public SpaceViolationPolicyEnforcement create( RegionServerServices rss, TableName tableName, SpaceQuotaSnapshot snapshot) { SpaceViolationPolicyEnforcement enforcement; SpaceQuotaStatus status = snapshot.getQuotaStatus(); if (!status.isInViolation()) { throw new IllegalArgumentException(tableName + " is not in violation. Snapshot=" + snapshot); } switch (status.getPolicy().get()) { case DISABLE: enforcement = new DisableTableViolationPolicyEnforcement(); break; case NO_WRITES_COMPACTIONS: enforcement = new NoWritesCompactionsViolationPolicyEnforcement(); break; case NO_WRITES: enforcement = new NoWritesViolationPolicyEnforcement(); break; case NO_INSERTS: enforcement = new NoInsertsViolationPolicyEnforcement(); break; default: throw new IllegalArgumentException("Unhandled SpaceViolationPolicy: " + status.getPolicy()); } enforcement.initialize(rss, tableName, snapshot); return enforcement; }
sizeToBeLoaded = enforcement.computeBulkLoadSize(regionServer.getFileSystem(), filePaths);
@Test public void testNoBulkLoadChecksOnNoSnapshot() { ActivePolicyEnforcement ape = new ActivePolicyEnforcement( new HashMap<TableName, SpaceViolationPolicyEnforcement>(), Collections.<TableName,SpaceQuotaSnapshot> emptyMap(), mock(RegionServerServices.class)); SpaceViolationPolicyEnforcement enforcement = ape.getPolicyEnforcement( TableName.valueOf("nonexistent")); assertFalse("Should not check bulkloads", enforcement.shouldCheckBulkLoads()); }
/** * Returns whether or not compactions should be disabled for the given <code>tableName</code> per * a space quota violation policy. A convenience method. * * @param tableName The table to check * @return True if compactions should be disabled for the table, false otherwise. */ public boolean areCompactionsDisabled(TableName tableName) { SpaceViolationPolicyEnforcement enforcement = this.enforcedPolicies.get(Objects.requireNonNull(tableName)); if (enforcement != null) { return enforcement.areCompactionsDisabled(); } return false; }
@Test(expected = SpaceLimitingException.class) public void testOneFileInBatchOverLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 1000L * 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 1024L); policy.initialize(rss, tableName, snapshot); policy.checkBulkLoad(fs, paths); }
@Test public void testExceptionOnPolicyEnforcementDisable() throws Exception { final TableName tableName = TableName.valueOf("foo"); final SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 2048L); RegionServerServices rss = mock(RegionServerServices.class); SpaceViolationPolicyEnforcementFactory factory = mock( SpaceViolationPolicyEnforcementFactory.class); SpaceViolationPolicyEnforcement enforcement = mock(SpaceViolationPolicyEnforcement.class); RegionServerSpaceQuotaManager realManager = new RegionServerSpaceQuotaManager(rss, factory); when(factory.create(rss, tableName, snapshot)).thenReturn(enforcement); doNothing().when(enforcement).enable(); doThrow(new IOException("Failed for test!")).when(enforcement).disable(); // Enabling should work realManager.enforceViolationPolicy(tableName, snapshot); Map<TableName, SpaceViolationPolicyEnforcement> enforcements = realManager.copyActiveEnforcements(); assertEquals(1, enforcements.size()); // If the disable fails, we should still treat it as "active" realManager.disableViolationPolicyEnforcement(tableName); enforcements = realManager.copyActiveEnforcements(); assertEquals(1, enforcements.size()); } }
@Override public boolean evaluate() throws Exception { ActivePolicyEnforcement enforcements = manager.getActiveEnforcements(); SpaceViolationPolicyEnforcement enforcement = enforcements.getPolicyEnforcement(tn); // Signifies that we're waiting on the quota snapshot to be fetched if (enforcement instanceof MissingSnapshotViolationPolicyEnforcement) { return false; } return enforcement.getQuotaSnapshot().getQuotaStatus().isInViolation(); } });
checkCellSizeLimit(region, mutation); spaceQuotaEnforcement.getPolicyEnforcement(region).check(mutation); quota.addMutation(mutation);
@Test public void testNoBulkLoadChecksOnNoSnapshot() { ActivePolicyEnforcement ape = new ActivePolicyEnforcement( new HashMap<TableName, SpaceViolationPolicyEnforcement>(), Collections.<TableName,SpaceQuotaSnapshot> emptyMap(), mock(RegionServerServices.class)); SpaceViolationPolicyEnforcement enforcement = ape.getPolicyEnforcement( TableName.valueOf("nonexistent")); assertFalse("Should not check bulkloads", enforcement.shouldCheckBulkLoads()); }
@Test public void testFilesUnderLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 100L * 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, length * 6); policy.initialize(rss, tableName, snapshot); policy.computeBulkLoadSize(fs, paths); }
@Test public void testFilesUnderLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 100L * 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, length * 6); policy.initialize(rss, tableName, snapshot); policy.checkBulkLoad(fs, paths); }
@Override public boolean evaluate() throws Exception { ActivePolicyEnforcement enforcements = manager.getActiveEnforcements(); SpaceViolationPolicyEnforcement enforcement = enforcements.getPolicyEnforcement(tn); // Signifies that we're waiting on the quota snapshot to be fetched if (enforcement instanceof MissingSnapshotViolationPolicyEnforcement) { return false; } return enforcement.getQuotaSnapshot().getQuotaStatus().isInViolation(); } });
@Test public void testExceptionOnPolicyEnforcementEnable() throws Exception { final TableName tableName = TableName.valueOf("foo"); final SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.DISABLE), 1024L, 2048L); RegionServerServices rss = mock(RegionServerServices.class); SpaceViolationPolicyEnforcementFactory factory = mock( SpaceViolationPolicyEnforcementFactory.class); SpaceViolationPolicyEnforcement enforcement = mock(SpaceViolationPolicyEnforcement.class); RegionServerSpaceQuotaManager realManager = new RegionServerSpaceQuotaManager(rss, factory); when(factory.create(rss, tableName, snapshot)).thenReturn(enforcement); doThrow(new IOException("Failed for test!")).when(enforcement).enable(); realManager.enforceViolationPolicy(tableName, snapshot); Map<TableName, SpaceViolationPolicyEnforcement> enforcements = realManager.copyActiveEnforcements(); assertTrue("Expected active enforcements to be empty, but were " + enforcements, enforcements.isEmpty()); }