@DataProvider(name = "offlineTagConfigTestDataProvider") public Object[][] offlineTagConfigTestDataProvider() { TableConfig.Builder tableConfigBuilder = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE); tableConfigBuilder.setTableName("testOfflineTable").setTimeColumnName("timeColumn").setTimeType("DAYS") .setRetentionTimeUnit("DAYS").setRetentionTimeValue("5").setServerTenant("aServerTenant"); List<Object[]> inputs = new ArrayList<>(); TableConfig tableConfig = tableConfigBuilder.build(); inputs.add(new Object[]{tableConfig, "aServerTenant", "aServerTenant_OFFLINE"}); tableConfig = tableConfigBuilder.setTagOverrideConfig(null).build(); inputs.add(new Object[]{tableConfig, "aServerTenant", "aServerTenant_OFFLINE"}); TagOverrideConfig tagOverrideConfig = new TagOverrideConfig(); tableConfig = tableConfigBuilder.setTagOverrideConfig(tagOverrideConfig).build(); inputs.add(new Object[]{tableConfig, "aServerTenant", "aServerTenant_OFFLINE"}); tagOverrideConfig = new TagOverrideConfig(); tagOverrideConfig.setRealtimeConsuming("overriddenTag_REALTIME"); tagOverrideConfig.setRealtimeCompleted("overriddenTag_OFFLINE"); tableConfig = tableConfigBuilder.setTagOverrideConfig(tagOverrideConfig).build(); inputs.add(new Object[]{tableConfig, "aServerTenant", "aServerTenant_OFFLINE"}); return inputs.toArray(new Object[inputs.size()][]); }
.setRetentionTimeUnit("DAYS").setRetentionTimeValue("5").setServerTenant("DefaultTenant");
TableConfig.Builder tableConfigBuilder = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE); tableConfigBuilder.setTableName("testRealtimeTable").setTimeColumnName("timeColumn").setTimeType("DAYS") .setRetentionTimeUnit("DAYS").setRetentionTimeValue("5").setServerTenant("aServerTenant");
TableConfig realtimeTableConfig = new TableConfig.Builder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); TableConfig offlineTableConfig = new TableConfig.Builder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build();
.setRetentionTimeUnit("DAYS").setRetentionTimeValue("5"); .constructStreamProperty(streamType, StreamConfigProperties.STREAM_CONSUMER_OFFSET_CRITERIA), "smallest"); _realtimeBuilder.setTableName(REALTIME_TABLE_NAME).setTimeColumnName("timeColumn").setTimeType("DAYS") .setRetentionTimeUnit("DAYS").setRetentionTimeValue("5").setSchemaName(REALTIME_TABLE_NAME) .setStreamConfigs(streamConfigs);
@BeforeClass public void setUp() throws Exception { try { startZk(); ControllerConf config = getDefaultControllerConfiguration(); config.setTableMinReplicas(MIN_NUM_REPLICAS); startController(config); ControllerRequestBuilderUtil .addFakeBrokerInstancesToAutoJoinHelixCluster(getHelixClusterName(), ZkStarter.DEFAULT_ZK_STR, NUM_BROKER_INSTANCES, true); ControllerRequestBuilderUtil .addFakeDataInstancesToAutoJoinHelixCluster(getHelixClusterName(), ZkStarter.DEFAULT_ZK_STR, NUM_SERVER_INSTANCES, true); _offlineBuilder.setTableName("testOfflineTable").setTimeColumnName("timeColumn").setTimeType("DAYS") .setRetentionTimeUnit("DAYS").setRetentionTimeValue("5"); setUpTable(); // Join 4 more servers as untagged String[] instanceNames = {"Server_localhost_a", "Server_localhost_b", "Server_localhost_c", "Server_localhost_d"}; for (String instanceName : instanceNames) { ControllerRequestBuilderUtil .addFakeDataInstanceToAutoJoinHelixCluster(getHelixClusterName(), ZkStarter.DEFAULT_ZK_STR, instanceName, true); _helixAdmin.removeInstanceTag(getHelixClusterName(), instanceName, OFFLINE_TENENT_NAME); } } catch (Exception e) { e.printStackTrace(); } }
@Test public void testOfflineTableWithNullQuotaButWithRealtimeTableConfigNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); TableConfig realtimeTableConfig = new TableConfig.Builder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setRealtimeTableConfig(_testPropertyStore, REALTIME_TABLE_NAME, TableConfig.toZnRecord(realtimeTableConfig)); ExternalView brokerResource = generateBrokerResource(OFFLINE_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(OFFLINE_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); // Nothing happened since it doesn't have qps quota. _tableQueryQuotaManager.dropTableQueryQuota(OFFLINE_TABLE_NAME); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
@Test public void testOfflineTableWithNullQuotaButWithRealtimeTableConfigNotNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); quotaConfig.setMaxQueriesPerSecond("100.00"); TableConfig realtimeTableConfig = new TableConfig.Builder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setRealtimeTableConfig(_testPropertyStore, REALTIME_TABLE_NAME, TableConfig.toZnRecord(realtimeTableConfig)); ExternalView brokerResource = generateBrokerResource(REALTIME_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(OFFLINE_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); // Drop the offline table won't have any affect since it is table type specific. _tableQueryQuotaManager.dropTableQueryQuota(OFFLINE_TABLE_NAME); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
@Test public void testRealtimeTableWithNullQuotaButWithOfflineTableConfigNotNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); quotaConfig.setMaxQueriesPerSecond("100.00"); TableConfig offlineTableConfig = new TableConfig.Builder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setOfflineTableConfig(_testPropertyStore, OFFLINE_TABLE_NAME, TableConfig.toZnRecord(offlineTableConfig)); ExternalView brokerResource = generateBrokerResource(OFFLINE_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(REALTIME_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
@Test public void testRealtimeTableWithNullQuotaButWithOfflineTableConfigNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); TableConfig offlineTableConfig = new TableConfig.Builder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setOfflineTableConfig(_testPropertyStore, OFFLINE_TABLE_NAME, TableConfig.toZnRecord(offlineTableConfig)); ExternalView brokerResource = generateBrokerResource(REALTIME_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(REALTIME_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
private TableConfig createRealtimeTableConfig1(int replicaCount) { return new TableConfig.Builder(CommonConstants.Helix.TableType.REALTIME).setTableName(TEST_TABLE_NAME).setLLC(true) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("5").setNumReplicas(replicaCount).build(); }
private TableConfig createOfflineTableConfig() throws Exception { return new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(TEST_TABLE_NAME) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("365").setNumReplicas(2).build(); }