public void setExistingTableConfig(TableConfig config, String tableNameWithType, TableType type) throws IOException { if (type == TableType.REALTIME) { ZKMetadataProvider.setRealtimeTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(config)); ensureRealtimeClusterIsSetUp(config, tableNameWithType, config.getIndexingConfig()); } else if (type == TableType.OFFLINE) { // Update replica group partition assignment to the property store if applicable updateReplicaGroupPartitionAssignment(config); ZKMetadataProvider.setOfflineTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(config)); IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableNameWithType); final String configReplication = config.getValidationConfig().getReplication(); if (configReplication != null && !config.getValidationConfig().getReplication() .equals(idealState.getReplicas())) { HelixHelper.updateIdealState(_helixZkManager, tableNameWithType, new Function<IdealState, IdealState>() { @Nullable @Override public IdealState apply(@Nullable IdealState idealState) { idealState.setReplicas(configReplication); return idealState; } }, RetryPolicies.exponentialBackoffRetryPolicy(5, 1000L, 1.2f)); } } }
private void addingTableToPropertyStore(String tableName) throws Exception { TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(tableName) .setTimeColumnName("timestamp").setTimeType("DAYS").build(); ZKMetadataProvider .setOfflineTableConfig(_propertyStore, tableConfig.getTableName(), TableConfig.toZnRecord(tableConfig)); } }
.setOfflineTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(tableConfig)); .setRealtimeTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(tableConfig));
@Test public void testRealtimeTableWithNullQuotaButWithOfflineTableConfigNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); TableConfig offlineTableConfig = new TableConfig.Builder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setOfflineTableConfig(_testPropertyStore, OFFLINE_TABLE_NAME, TableConfig.toZnRecord(offlineTableConfig)); ExternalView brokerResource = generateBrokerResource(REALTIME_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(REALTIME_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
@Test public void testOfflineTableWithNullQuotaButWithRealtimeTableConfigNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); TableConfig realtimeTableConfig = new TableConfig.Builder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setRealtimeTableConfig(_testPropertyStore, REALTIME_TABLE_NAME, TableConfig.toZnRecord(realtimeTableConfig)); ExternalView brokerResource = generateBrokerResource(OFFLINE_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(OFFLINE_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); // Nothing happened since it doesn't have qps quota. _tableQueryQuotaManager.dropTableQueryQuota(OFFLINE_TABLE_NAME); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
@Test public void testOfflineTableWithNullQuotaButWithRealtimeTableConfigNotNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); quotaConfig.setMaxQueriesPerSecond("100.00"); TableConfig realtimeTableConfig = new TableConfig.Builder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setRealtimeTableConfig(_testPropertyStore, REALTIME_TABLE_NAME, TableConfig.toZnRecord(realtimeTableConfig)); ExternalView brokerResource = generateBrokerResource(REALTIME_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(OFFLINE_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); // Drop the offline table won't have any affect since it is table type specific. _tableQueryQuotaManager.dropTableQueryQuota(OFFLINE_TABLE_NAME); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
@Test public void testRealtimeTableWithNullQuotaButWithOfflineTableConfigNotNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); quotaConfig.setMaxQueriesPerSecond("100.00"); TableConfig offlineTableConfig = new TableConfig.Builder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setOfflineTableConfig(_testPropertyStore, OFFLINE_TABLE_NAME, TableConfig.toZnRecord(offlineTableConfig)); ExternalView brokerResource = generateBrokerResource(OFFLINE_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(REALTIME_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
ZNRecord znRecord = TableConfig.toZnRecord(tableConfig); tableConfigToCompare = TableConfig.fromZnRecord(znRecord); Assert.assertEquals(tableConfigToCompare.getTableName(), tableConfig.getTableName()); ZNRecord znRecord = TableConfig.toZnRecord(tableConfig); tableConfigToCompare = TableConfig.fromZnRecord(znRecord); Assert.assertEquals(tableConfigToCompare.getTableName(), tableConfig.getTableName()); ZNRecord znRecord = TableConfig.toZnRecord(tableConfig); tableConfigToCompare = TableConfig.fromZnRecord(znRecord); Assert.assertEquals(tableConfigToCompare.getTableName(), tableConfig.getTableName()); znRecord = TableConfig.toZnRecord(tableConfig); tableConfigToCompare = TableConfig.fromZnRecord(znRecord); Assert.assertEquals(tableConfigToCompare.getTableName(), tableConfig.getTableName()); ZNRecord znRecord = TableConfig.toZnRecord(tableConfig); tableConfigToCompare = TableConfig.fromZnRecord(znRecord); checkTableConfigWithAssignmentConfig(tableConfig, tableConfigToCompare); ZNRecord znRecord = TableConfig.toZnRecord(tableConfig); tableConfigToCompare = TableConfig.fromZnRecord(znRecord); Assert.assertEquals( ZNRecord znRecord = TableConfig.toZnRecord(tableConfig); tableConfigToCompare = TableConfig.fromZnRecord(znRecord); checkTableConfigWithStarTreeConfig(tableConfig, tableConfigToCompare); ZNRecord znRecord = TableConfig.toZnRecord(tableConfig);
.setRealtimeTableConfig(_testPropertyStore, REALTIME_TABLE_NAME, TableConfig.toZnRecord(realtimeTableConfig)); ZKMetadataProvider .setOfflineTableConfig(_testPropertyStore, OFFLINE_TABLE_NAME, TableConfig.toZnRecord(offlineTableConfig));