private void setQps(TableConfig tableConfig) { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setMaxQueriesPerSecond("100.00"); tableConfig.setQuotaConfig(quotaConfig); }
if (quotaConfig == null || Strings.isNullOrEmpty(quotaConfig.getMaxQueriesPerSecond())) { LOGGER.info("No qps config specified for table: {}", tableNameWithType); return; if (quotaConfig.isMaxQueriesPerSecondValid()) { overallRate = Double.parseDouble(quotaConfig.getMaxQueriesPerSecond()); } else { LOGGER.error("Failed to init qps quota: error when parsing qps quota: {} for table: {}", quotaConfig.getMaxQueriesPerSecond(), tableNameWithType); return;
@Test public void testBadQuotaConfig() throws IOException { { String quotaConfigStr = "{\"storage\" : \"124GB3GB\"}"; QuotaConfig quotaConfig = JsonUtils.stringToObject(quotaConfigStr, QuotaConfig.class); Assert.assertNotNull(quotaConfig.getStorage()); Assert.assertEquals(quotaConfig.storageSizeBytes(), -1); } }
public void validate() { if (!isStorageValid()) { LOGGER.error("Failed to convert storage quota config: {} to bytes", _storage); throw new ConfigurationRuntimeException("Failed to convert storage quota config: " + _storage + " to bytes"); } if (!isMaxQueriesPerSecondValid()) { LOGGER.error("Failed to convert qps quota config: {}", _maxQueriesPerSecond); throw new ConfigurationRuntimeException("Failed to convert qps quota config: " + _maxQueriesPerSecond); } }
@Test(expectedExceptions = ConfigurationRuntimeException.class) public void testNegativeQpsQuota() throws IOException { String quotaConfigStr = "{\"maxQueriesPerSecond\" : \"-1.0\"}"; QuotaConfig quotaConfig = JsonUtils.stringToObject(quotaConfigStr, QuotaConfig.class); Assert.assertNotNull(quotaConfig.getMaxQueriesPerSecond()); quotaConfig.validate(); }
Assert.assertNull(tableConfig.getQuotaConfig()); QuotaConfig quota = new QuotaConfig(); quota.setStorage("10G"); tableConfig.setQuotaConfig(quota); sendPutRequest(_controllerRequestURLBuilder.forUpdateTableConfig(tableName), tableConfig.toJSONConfigString()); modifiedConfig = getTableConfig(tableName, "REALTIME"); Assert.assertNotNull(modifiedConfig.getQuotaConfig()); Assert.assertEquals(modifiedConfig.getQuotaConfig().getStorage(), "10G"); Assert.assertNull(modifiedConfig.getQuotaConfig().getMaxQueriesPerSecond()); quota.setMaxQueriesPerSecond("100.00"); tableConfig.setQuotaConfig(quota); sendPutRequest(_controllerRequestURLBuilder.forUpdateTableConfig(tableName), tableConfig.toJSONConfigString()); modifiedConfig = getTableConfig(tableName, "REALTIME"); Assert.assertNotNull(modifiedConfig.getQuotaConfig().getMaxQueriesPerSecond()); Assert.assertEquals(modifiedConfig.getQuotaConfig().getMaxQueriesPerSecond(), "100.00");
@Test public void testRealtimeTableWithNullQuotaButWithOfflineTableConfigNotNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); quotaConfig.setMaxQueriesPerSecond("100.00"); TableConfig offlineTableConfig = new TableConfig.Builder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setOfflineTableConfig(_testPropertyStore, OFFLINE_TABLE_NAME, TableConfig.toZnRecord(offlineTableConfig)); ExternalView brokerResource = generateBrokerResource(OFFLINE_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(REALTIME_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
@Test public void testRealtimeTableWithNullQuotaButWithOfflineTableConfigNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); TableConfig offlineTableConfig = new TableConfig.Builder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setOfflineTableConfig(_testPropertyStore, OFFLINE_TABLE_NAME, TableConfig.toZnRecord(offlineTableConfig)); ExternalView brokerResource = generateBrokerResource(REALTIME_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(REALTIME_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
if ((offlineQuotaConfig != null && !Strings.isNullOrEmpty(offlineQuotaConfig.getMaxQueriesPerSecond())) && ( realtimeQuotaConfig != null && !Strings.isNullOrEmpty(realtimeQuotaConfig.getMaxQueriesPerSecond()))) { if (!offlineQuotaConfig.getMaxQueriesPerSecond().equals(realtimeQuotaConfig.getMaxQueriesPerSecond())) { LOGGER.warn( "Attention! The values of MaxQueriesPerSecond for table {} are set different! Offline table qps quota: {}, Real-time table qps quota: {}", rawTableName, offlineQuotaConfig.getMaxQueriesPerSecond(), realtimeQuotaConfig.getMaxQueriesPerSecond());
@Test public void testNoStorageQuotaConfig() throws InvalidConfigException { StorageQuotaChecker checker = new MockStorageQuotaChecker(_tableConfig, _tableSizeReader, _controllerMetrics, _pinotHelixResourceManager); when(_tableConfig.getQuotaConfig()).thenReturn(_quotaConfig); when(_quotaConfig.storageSizeBytes()).thenReturn(-1L); StorageQuotaChecker.QuotaCheckerResponse res = checker.isSegmentStorageWithinQuota(TEST_DIR, "myTable", "segment", 1000); Assert.assertTrue(res.isSegmentWithinQuota); }
@Test(expectedExceptions = ConfigurationRuntimeException.class) public void testBadConfig() throws IOException { String quotaConfigStr = "{\"storage\":\"-1M\"}"; QuotaConfig quotaConfig = JsonUtils.stringToObject(quotaConfigStr, QuotaConfig.class); quotaConfig.validate(); }
private void validateLoadedConfig(TableConfig config) { Assert.assertEquals(config.getTableName(), "mytable_OFFLINE"); Assert.assertEquals(config.getQuotaConfig().getStorage(), "125 GiB"); Assert.assertEquals(config.getValidationConfig().getRetentionTimeValue(), "5"); Assert.assertEquals(config.getValidationConfig().getRetentionTimeUnit(), "DAYS"); Assert.assertEquals(config.getTenantConfig().getBroker(), "foo"); Assert.assertEquals(config.getTenantConfig().getServer(), "bar"); Assert.assertEquals(config.getIndexingConfig().getSortedColumn(), Lists.newArrayList("foo")); } }
QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("30G"); TableConfig tableConfig = tableConfigBuilder.setQuotaConfig(quotaConfig).build(); Assert.assertEquals(tableConfig.getIndexingConfig().getLoadMode(), "HEAP"); Assert.assertNotNull(tableConfig.getQuotaConfig()); Assert.assertEquals(tableConfig.getQuotaConfig().getStorage(), "30G"); Assert.assertNull(tableConfig.getQuotaConfig().getMaxQueriesPerSecond()); quotaConfig.setMaxQueriesPerSecond("100.00"); tableConfig = tableConfigBuilder.setQuotaConfig(quotaConfig).build(); Assert.assertNotNull(tableConfig.getQuotaConfig()); Assert.assertNotNull(tableConfig.getQuotaConfig().getMaxQueriesPerSecond()); Assert.assertEquals(tableConfig.getQuotaConfig().getMaxQueriesPerSecond(), "100.00"); Assert.assertNotNull(tableConfigToCompare.getQuotaConfig()); Assert .assertEquals(tableConfigToCompare.getQuotaConfig().getStorage(), tableConfig.getQuotaConfig().getStorage()); Assert.assertNotNull(tableConfigToCompare.getQuotaConfig()); Assert .assertEquals(tableConfigToCompare.getQuotaConfig().getStorage(), tableConfig.getQuotaConfig().getStorage());
@Test public void testOfflineTableWithNullQuotaButWithRealtimeTableConfigNotNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); quotaConfig.setMaxQueriesPerSecond("100.00"); TableConfig realtimeTableConfig = new TableConfig.Builder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setRealtimeTableConfig(_testPropertyStore, REALTIME_TABLE_NAME, TableConfig.toZnRecord(realtimeTableConfig)); ExternalView brokerResource = generateBrokerResource(REALTIME_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(OFFLINE_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); // Drop the offline table won't have any affect since it is table type specific. _tableQueryQuotaManager.dropTableQueryQuota(OFFLINE_TABLE_NAME); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
@Test(expectedExceptions = ConfigurationRuntimeException.class) public void testInvalidQpsQuota() throws IOException { String quotaConfigStr = "{\"maxQueriesPerSecond\" : \"InvalidQpsQuota\"}"; QuotaConfig quotaConfig = JsonUtils.stringToObject(quotaConfigStr, QuotaConfig.class); Assert.assertNotNull(quotaConfig.getMaxQueriesPerSecond()); quotaConfig.validate(); }
@Test public void testOfflineTableWithNullQuotaButWithRealtimeTableConfigNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); TableConfig realtimeTableConfig = new TableConfig.Builder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setRealtimeTableConfig(_testPropertyStore, REALTIME_TABLE_NAME, TableConfig.toZnRecord(realtimeTableConfig)); ExternalView brokerResource = generateBrokerResource(OFFLINE_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(OFFLINE_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); // Nothing happened since it doesn't have qps quota. _tableQueryQuotaManager.dropTableQueryQuota(OFFLINE_TABLE_NAME); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
@Nonnull public static TableConfig fromJSONConfig(@Nonnull JsonNode jsonConfig) throws IOException { TableType tableType = TableType.valueOf(jsonConfig.get(TABLE_TYPE_KEY).asText().toUpperCase()); String tableName = TableNameBuilder.forType(tableType).tableNameWithType(jsonConfig.get(TABLE_NAME_KEY).asText()); SegmentsValidationAndRetentionConfig validationConfig = extractChildConfig(jsonConfig, VALIDATION_CONFIG_KEY, SegmentsValidationAndRetentionConfig.class); TenantConfig tenantConfig = extractChildConfig(jsonConfig, TENANT_CONFIG_KEY, TenantConfig.class); IndexingConfig indexingConfig = extractChildConfig(jsonConfig, INDEXING_CONFIG_KEY, IndexingConfig.class); TableCustomConfig customConfig = extractChildConfig(jsonConfig, CUSTOM_CONFIG_KEY, TableCustomConfig.class); QuotaConfig quotaConfig = null; if (jsonConfig.has(QUOTA_CONFIG_KEY)) { quotaConfig = extractChildConfig(jsonConfig, QUOTA_CONFIG_KEY, QuotaConfig.class); quotaConfig.validate(); } TableTaskConfig taskConfig = null; if (jsonConfig.has(TASK_CONFIG_KEY)) { taskConfig = extractChildConfig(jsonConfig, TASK_CONFIG_KEY, TableTaskConfig.class); } RoutingConfig routingConfig = null; if (jsonConfig.has(ROUTING_CONFIG_KEY)) { routingConfig = extractChildConfig(jsonConfig, ROUTING_CONFIG_KEY, RoutingConfig.class); } return new TableConfig(tableName, tableType, validationConfig, tenantConfig, indexingConfig, customConfig, quotaConfig, taskConfig, routingConfig); }
brokerResource.setState(REALTIME_TABLE_NAME, "broker_instance_2", "OFFLINE"); QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); quotaConfig.setMaxQueriesPerSecond("100.00"); TableConfig realtimeTableConfig = new TableConfig.Builder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setBrokerTenant("testBroker").setServerTenant("testServer").build(); QuotaConfig quotaConfig2 = new QuotaConfig(); quotaConfig2.setStorage("6G"); quotaConfig2.setMaxQueriesPerSecond("100.00"); TableConfig offlineTableConfig = new TableConfig.Builder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig)
@Test public void testQuotaConfig() throws IOException { { String quotaConfigStr = "{\"storage\" : \"100g\"}"; QuotaConfig quotaConfig = JsonUtils.stringToObject(quotaConfigStr, QuotaConfig.class); Assert.assertEquals(quotaConfig.getStorage(), "100g"); Assert.assertEquals(quotaConfig.storageSizeBytes(), 100 * 1024 * 1024 * 1024L); } { String quotaConfigStr = "{}"; QuotaConfig quotaConfig = JsonUtils.stringToObject(quotaConfigStr, QuotaConfig.class); Assert.assertNull(quotaConfig.getStorage()); Assert.assertEquals(quotaConfig.storageSizeBytes(), -1); } }
@Test public void testInvalidQpsQuota() throws Exception { ExternalView brokerResource = generateBrokerResource(OFFLINE_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(OFFLINE_TABLE_NAME); // Set invalid qps quota QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setMaxQueriesPerSecond("InvalidQpsQuota"); tableConfig.setQuotaConfig(quotaConfig); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }