@Test public void testBadQuotaConfig() throws IOException { { String quotaConfigStr = "{\"storage\" : \"124GB3GB\"}"; QuotaConfig quotaConfig = JsonUtils.stringToObject(quotaConfigStr, QuotaConfig.class); Assert.assertNotNull(quotaConfig.getStorage()); Assert.assertEquals(quotaConfig.storageSizeBytes(), -1); } }
@Test public void testNoStorageQuotaConfig() throws InvalidConfigException { StorageQuotaChecker checker = new MockStorageQuotaChecker(_tableConfig, _tableSizeReader, _controllerMetrics, _pinotHelixResourceManager); when(_tableConfig.getQuotaConfig()).thenReturn(_quotaConfig); when(_quotaConfig.storageSizeBytes()).thenReturn(-1L); StorageQuotaChecker.QuotaCheckerResponse res = checker.isSegmentStorageWithinQuota(TEST_DIR, "myTable", "segment", 1000); Assert.assertTrue(res.isSegmentWithinQuota); }
@Test public void testQuotaConfig() throws IOException { { String quotaConfigStr = "{\"storage\" : \"100g\"}"; QuotaConfig quotaConfig = JsonUtils.stringToObject(quotaConfigStr, QuotaConfig.class); Assert.assertEquals(quotaConfig.getStorage(), "100g"); Assert.assertEquals(quotaConfig.storageSizeBytes(), 100 * 1024 * 1024 * 1024L); } { String quotaConfigStr = "{}"; QuotaConfig quotaConfig = JsonUtils.stringToObject(quotaConfigStr, QuotaConfig.class); Assert.assertNull(quotaConfig.getStorage()); Assert.assertEquals(quotaConfig.storageSizeBytes(), -1); } }
when(_tableConfig.getTableName()).thenReturn(tableName); when(_tableConfig.getQuotaConfig()).thenReturn(_quotaConfig); when(_quotaConfig.storageSizeBytes()).thenReturn(3000L); when(_quotaConfig.getStorage()).thenReturn("3K"); StorageQuotaChecker checker = when(_quotaConfig.storageSizeBytes()).thenReturn(2800L); when(_quotaConfig.getStorage()).thenReturn("2.8K"); response = checker.isSegmentStorageWithinQuota(TEST_DIR, tableName, "segment1", 1000); when(_quotaConfig.storageSizeBytes()).thenReturn(2800L); when(_quotaConfig.getStorage()).thenReturn("2.8K"); response = checker.isSegmentStorageWithinQuota(TEST_DIR, tableName, "segment1", 1000); when(_quotaConfig.storageSizeBytes()).thenReturn(2800L); when(_quotaConfig.getStorage()).thenReturn("2.8K"); response = checker.isSegmentStorageWithinQuota(TEST_DIR, tableName, "segment1", 1000); when(_quotaConfig.storageSizeBytes()).thenReturn(2800L); when(_quotaConfig.getStorage()).thenReturn("2.8K"); response = checker.isSegmentStorageWithinQuota(TEST_DIR, tableName, "segment1", 1000); when(_quotaConfig.storageSizeBytes()).thenReturn(2800L); when(_quotaConfig.getStorage()).thenReturn("2.8K"); response = checker.isSegmentStorageWithinQuota(TEST_DIR, tableName, "segment1", 1000);
long allowedStorageBytes = numReplicas * quotaConfig.storageSizeBytes(); if (allowedStorageBytes < 0) { LOGGER.warn("Storage quota is not configured for table: {}", tableNameWithType); "Appending Segment %s of Table %s is within quota. Total allowed storage size: %s ( = configured quota: %s * number replicas: %d). New estimated table size of all replicas: %s. Current table size of all replicas: %s. Incoming uncompressed segment size of all replicas: %s ( = single incoming uncompressed segment size: %s * number replicas: %d). Formula: New estimated size = current table size + incoming segment size", segmentName, tableName, DataSize.fromBytes(allowedStorageBytes), DataSize.fromBytes(quotaConfig.storageSizeBytes()), numReplicas, DataSize.fromBytes(estimatedFinalSizeBytes), DataSize.fromBytes(tableSubtypeSize.estimatedSizeInBytes), DataSize.fromBytes(totalIncomingSegmentSizeBytes), DataSize.fromBytes(incomingSegmentSizeBytes), "Refreshing Segment %s of Table %s is within quota. Total allowed storage size: %s ( = configured quota: %s * number replicas: %d). New estimated table size of all replicas: %s. Current table size of all replicas: %s. Incoming uncompressed segment size of all replicas: %s ( = single incoming uncompressed segment size: %s * number replicas: %d). Existing same segment size of all replicas: %s. Formula: New estimated size = current table size - existing same segment size + incoming segment size", segmentName, tableName, DataSize.fromBytes(allowedStorageBytes), DataSize.fromBytes(quotaConfig.storageSizeBytes()), numReplicas, DataSize.fromBytes(estimatedFinalSizeBytes), DataSize.fromBytes(tableSubtypeSize.estimatedSizeInBytes), DataSize.fromBytes(totalIncomingSegmentSizeBytes), DataSize.fromBytes(incomingSegmentSizeBytes), "Table %s already over quota. Existing estimated uncompressed table size of all replicas: %s > total allowed storage size: %s ( = configured quota: %s * num replicas: %d). Check if indexes were enabled recently and adjust table quota accordingly.", tableName, DataSize.fromBytes(tableSubtypeSize.estimatedSizeInBytes), DataSize.fromBytes(allowedStorageBytes), DataSize.fromBytes(quotaConfig.storageSizeBytes()), numReplicas); } else { message = String.format( DataSize.fromBytes(tableSubtypeSize.estimatedSizeInBytes), DataSize.fromBytes(existingSegmentSizeBytes), DataSize.fromBytes(incomingSegmentSizeBytes), numReplicas, DataSize.fromBytes(quotaConfig.storageSizeBytes()), numReplicas);