@Test(expected = SpaceLimitingException.class) public void testSumOfFilesOverLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now, but 5*1024 files would push us to violation SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 5000L); policy.initialize(rss, tableName, snapshot); policy.computeBulkLoadSize(fs, paths); } }
@Test(expected = IllegalArgumentException.class) public void testFileIsNotAFile() throws Exception { final List<String> paths = new ArrayList<>(); String path = "/1"; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(1000L); when(status.isFile()).thenReturn(false); paths.add(path); // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, Long.MAX_VALUE); policy.initialize(rss, tableName, snapshot); // If the file to bulk load isn't a file, this should throw an exception policy.computeBulkLoadSize(fs, paths); }
@Test public void testFilesUnderLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 100L * 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, length * 6); policy.initialize(rss, tableName, snapshot); policy.computeBulkLoadSize(fs, paths); }
SpaceQuotaStatus.notInViolation(), 1024L * 768L, 1024L * 1024L); .setEndKey(Bytes.toBytes(3)) .build(), 1024L * 256L); tn1Snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 1024L * 1024L, 1024L * 1024L);
policiesToEnforce.put( TableName.valueOf("table3"), new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 256L, 512L)); policiesToEnforce.put( TableName.valueOf("table4"), new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 128L, 512L));
@Test(expected = SpaceLimitingException.class) public void testSumOfFilesOverLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now, but 5*1024 files would push us to violation SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 5000L); policy.initialize(rss, tableName, snapshot); policy.checkBulkLoad(fs, paths); } }
@Test(expected = SpaceLimitingException.class) public void testOneFileInBatchOverLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 1000L * 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 1024L); policy.initialize(rss, tableName, snapshot); policy.checkBulkLoad(fs, paths); }
@Test public void testFilesUnderLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 100L * 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, length * 6); policy.initialize(rss, tableName, snapshot); policy.checkBulkLoad(fs, paths); }
@Test public void testNonViolatingQuotaCachesPolicyEnforcment() { final Map<TableName,SpaceQuotaSnapshot> snapshots = new HashMap<>(); final TableName tableName = TableName.valueOf("my_table"); snapshots.put(tableName, new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 1024)); final ActivePolicyEnforcement ape = new ActivePolicyEnforcement( Collections.emptyMap(), snapshots, rss); SpaceViolationPolicyEnforcement policyEnforcement = ape.getPolicyEnforcement(tableName); assertTrue( "Found the wrong class: " + policyEnforcement.getClass(), policyEnforcement instanceof DefaultViolationPolicyEnforcement); SpaceViolationPolicyEnforcement copy = ape.getPolicyEnforcement(tableName); assertTrue("Expected the instance to be cached", policyEnforcement == copy); Entry<TableName,SpaceViolationPolicyEnforcement> entry = ape.getLocallyCachedPolicies().entrySet().iterator().next(); assertTrue(policyEnforcement == entry.getValue()); }
@Test(expected = IllegalArgumentException.class) public void testFileIsNotAFile() throws Exception { final List<String> paths = new ArrayList<>(); String path = "/1"; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(1000L); when(status.isFile()).thenReturn(false); paths.add(path); // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, Long.MAX_VALUE); policy.initialize(rss, tableName, snapshot); // If the file to bulk load isn't a file, this should throw an exception policy.checkBulkLoad(fs, paths); }
@Test public void testQuotaSnapshotConversion() { MetricsMasterWrapperImpl info = new MetricsMasterWrapperImpl( TEST_UTIL.getHBaseCluster().getMaster()); assertEquals(new SimpleImmutableEntry<Long,Long>(1024L, 2048L), info.convertSnapshot(new SpaceQuotaSnapshot( SpaceQuotaStatus.notInViolation(), 1024L, 2048L))); assertEquals(new SimpleImmutableEntry<Long,Long>(4096L, 2048L), info.convertSnapshot(new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 2048L))); } }
SpaceQuotaStatus.notInViolation(), currentSnapshot.getUsage(), currentSnapshot.getLimit()); this.snapshotNotifier.transitionTable(tableInLimbo, targetSnapshot);
SpaceQuotaStatus.notInViolation(), 1024L * 768L, 1024L * 1024L); .setEndKey(Bytes.toBytes(3)) .build(), 1024L * 256L); tn1Snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 1024L * 1024L, 1024L * 1024L);
policiesToEnforce.put( TableName.valueOf("table3"), new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 256L, 512L)); policiesToEnforce.put( TableName.valueOf("table4"), new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 128L, 512L));
@Override public SpaceQuotaSnapshot getTargetState( String subject, SpaceQuota spaceQuota) throws IOException { rlock.lock(); try { final long sizeLimitInBytes = spaceQuota.getSoftLimit(); long sum = 0L; for (Entry<RegionInfo,Long> entry : filterBySubject(subject)) { sum += entry.getValue(); } // Add in the size for any snapshots against this table sum += QuotaTableUtil.getNamespaceSnapshotSize(conn, subject); // Observance is defined as the size of the table being less than the limit SpaceQuotaStatus status = sum <= sizeLimitInBytes ? SpaceQuotaStatus.notInViolation() : new SpaceQuotaStatus(ProtobufUtil.toViolationPolicy(spaceQuota.getViolationPolicy())); return new SpaceQuotaSnapshot(status, sum, sizeLimitInBytes); } finally { rlock.unlock(); } }
@Test(expected = SpaceLimitingException.class) public void testOneFileInBatchOverLimit() throws Exception { final List<String> paths = new ArrayList<>(); final List<FileStatus> statuses = new ArrayList<>(); final long length = 1000L * 1024L; for (int i = 0; i < 5; i++) { String path = "/" + i; FileStatus status = mock(FileStatus.class); when(fs.getFileStatus(new Path(path))).thenReturn(status); when(status.getLen()).thenReturn(length); when(status.isFile()).thenReturn(true); paths.add(path); statuses.add(status); } // Quota is not in violation now SpaceQuotaSnapshot snapshot = new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 1024L); policy.initialize(rss, tableName, snapshot); policy.computeBulkLoadSize(fs, paths); }
@Test public void testNonViolatingQuotaCachesPolicyEnforcment() { final Map<TableName,SpaceQuotaSnapshot> snapshots = new HashMap<>(); final TableName tableName = TableName.valueOf("my_table"); snapshots.put(tableName, new SpaceQuotaSnapshot(SpaceQuotaStatus.notInViolation(), 0, 1024)); final ActivePolicyEnforcement ape = new ActivePolicyEnforcement( Collections.emptyMap(), snapshots, rss); SpaceViolationPolicyEnforcement policyEnforcement = ape.getPolicyEnforcement(tableName); assertTrue( "Found the wrong class: " + policyEnforcement.getClass(), policyEnforcement instanceof DefaultViolationPolicyEnforcement); SpaceViolationPolicyEnforcement copy = ape.getPolicyEnforcement(tableName); assertTrue("Expected the instance to be cached", policyEnforcement == copy); Entry<TableName,SpaceViolationPolicyEnforcement> entry = ape.getLocallyCachedPolicies().entrySet().iterator().next(); assertTrue(policyEnforcement == entry.getValue()); }
@Override public SpaceQuotaSnapshot getTargetState( TableName table, SpaceQuota spaceQuota) throws IOException { rlock.lock(); try { final long sizeLimitInBytes = spaceQuota.getSoftLimit(); long sum = 0L; for (Entry<RegionInfo,Long> entry : filterBySubject(table)) { sum += entry.getValue(); } // Add in the size for any snapshots against this table sum += getSnapshotSizesForTable(table); // Observance is defined as the size of the table being less than the limit SpaceQuotaStatus status = sum <= sizeLimitInBytes ? SpaceQuotaStatus.notInViolation() : new SpaceQuotaStatus(ProtobufUtil.toViolationPolicy(spaceQuota.getViolationPolicy())); return new SpaceQuotaSnapshot(status, sum, sizeLimitInBytes); } finally { rlock.unlock(); } }
@Test public void testQuotaSnapshotConversion() { MetricsMasterWrapperImpl info = new MetricsMasterWrapperImpl( TEST_UTIL.getHBaseCluster().getMaster()); assertEquals(new SimpleImmutableEntry<Long,Long>(1024L, 2048L), info.convertSnapshot(new SpaceQuotaSnapshot( SpaceQuotaStatus.notInViolation(), 1024L, 2048L))); assertEquals(new SimpleImmutableEntry<Long,Long>(4096L, 2048L), info.convertSnapshot(new SpaceQuotaSnapshot( new SpaceQuotaStatus(SpaceViolationPolicy.NO_INSERTS), 4096L, 2048L))); }