@BeforeClass public static void setup() throws Exception { instanceZKMetadata = getInstanceZKMetadata(); realtimeSegmentZKMetadata = getRealtimeSegmentZKMetadata(); tableDataManagerConfig = getTableDataManagerConfig(); Map<String, String> streamConfigs = new HashMap<>(); streamConfigs.put("streamType", "kafka"); streamConfigs.put("stream.kafka.consumer.type", "highLevel"); streamConfigs.put("stream.kafka.topic.name", "kafkaTopic"); streamConfigs .put("stream.kafka.decoder.class.name", "org.apache.pinot.core.realtime.impl.kafka.KafkaAvroMessageDecoder"); streamConfigs.put("stream.kafka.hlc.zk.connect.string", "localhost:1111/zkConnect"); streamConfigs.put("stream.kafka.decoder.prop.schema.registry.rest.url", "http://localhost:2222/schemaRegistry"); tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.REALTIME).setTableName("mirror") .setStreamConfigs(streamConfigs).build(); }
@DataProvider(name = "offlineTagConfigTestDataProvider") public Object[][] offlineTagConfigTestDataProvider() { TableConfig.Builder tableConfigBuilder = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE); tableConfigBuilder.setTableName("testOfflineTable").setTimeColumnName("timeColumn").setTimeType("DAYS") .setRetentionTimeUnit("DAYS").setRetentionTimeValue("5").setServerTenant("aServerTenant"); List<Object[]> inputs = new ArrayList<>(); TableConfig tableConfig = tableConfigBuilder.build(); inputs.add(new Object[]{tableConfig, "aServerTenant", "aServerTenant_OFFLINE"}); tableConfig = tableConfigBuilder.setTagOverrideConfig(null).build(); inputs.add(new Object[]{tableConfig, "aServerTenant", "aServerTenant_OFFLINE"}); TagOverrideConfig tagOverrideConfig = new TagOverrideConfig(); tableConfig = tableConfigBuilder.setTagOverrideConfig(tagOverrideConfig).build(); inputs.add(new Object[]{tableConfig, "aServerTenant", "aServerTenant_OFFLINE"}); tagOverrideConfig = new TagOverrideConfig(); tagOverrideConfig.setRealtimeConsuming("overriddenTag_REALTIME"); tagOverrideConfig.setRealtimeCompleted("overriddenTag_OFFLINE"); tableConfig = tableConfigBuilder.setTagOverrideConfig(tagOverrideConfig).build(); inputs.add(new Object[]{tableConfig, "aServerTenant", "aServerTenant_OFFLINE"}); return inputs.toArray(new Object[inputs.size()][]); }
public static TableConfig getDynamicComputingTableConfig(String tableName) { TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName); RoutingConfig routingConfig = new RoutingConfig(); Map<String, String> routingTableBuilderOptions = new HashMap<>(); routingTableBuilderOptions.put(RoutingConfig.ENABLE_DYNAMIC_COMPUTING_KEY, "true"); routingConfig.setRoutingTableBuilderOptions(routingTableBuilderOptions); return new TableConfig.Builder(tableType).setTableName(tableName).setRoutingConfig(routingConfig).build(); } }
private void setUpTable() throws Exception { // Create table config TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(TABLE_NAME).setNumReplicas(2) .setSegmentAssignmentStrategy("RandomAssignmentStrategy").build(); // Create the table and upload segments _helixResourceManager.addTable(tableConfig); // Wait for table addition while (!_helixResourceManager.hasOfflineTable(TABLE_NAME)) { Thread.sleep(100); } // Upload segments ReplicaGroupTestUtils.uploadMultipleSegmentsWithPartitionNumber(TABLE_NAME, INITIAL_NUM_SEGMENTS, PARTITION_COLUMN, _helixResourceManager, 1); // Wait for all segments appear in the external view while (!allSegmentsPushedToIdealState(TABLE_NAME, INITIAL_NUM_SEGMENTS)) { Thread.sleep(100); } }
@BeforeClass public void setUp() throws Exception { _zookeeperInstance = ZkStarter.startLocalZkServer(); _zkClient = new ZkClient(ZK_STR); Thread.sleep(1000); _pinotHelixResourceManager = new PinotHelixResourceManager(ZK_STR, HELIX_CLUSTER_NAME, CONTROLLER_INSTANCE_NAME, null, 1000L, true, /*isUpdateStateModel=*/ false, true); _pinotHelixResourceManager.start(); ControllerRequestBuilderUtil.addFakeDataInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_STR, 2, true); ControllerRequestBuilderUtil.addFakeBrokerInstancesToAutoJoinHelixCluster(HELIX_CLUSTER_NAME, ZK_STR, 2, true); _offlineTableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(TEST_TABLE_NAME).setNumReplicas(2) .build(); final String instanceId = "localhost_helixController"; _helixManager = HelixSetupUtils.setup(HELIX_CLUSTER_NAME, ZK_STR, instanceId, /*isUpdateStateModel=*/false, true); _pinotHelixResourceManager.addTable(_offlineTableConfig); }
private TableConfig buildRealtimeTableConfig() throws Exception { // Create partition config Map<String, ColumnPartitionConfig> metadataMap = new HashMap<>(); metadataMap.put(PARTITION_COLUMN, new ColumnPartitionConfig(PARTITION_FUNCTION_NAME, NUM_PARTITION)); SegmentPartitionConfig partitionConfig = new SegmentPartitionConfig(metadataMap); // Create the routing config RoutingConfig routingConfig = new RoutingConfig(); routingConfig.setRoutingTableBuilderName("PartitionAwareOffline"); // Create table config TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.REALTIME).setTableName(REALTIME_TABLE_NAME) .setNumReplicas(NUM_REPLICA).build(); tableConfig.getValidationConfig().setReplicasPerPartition(Integer.toString(NUM_REPLICA)); tableConfig.getIndexingConfig().setSegmentPartitionConfig(partitionConfig); tableConfig.setRoutingConfig(routingConfig); return tableConfig; }
private void addTable() throws Exception { if (_tableConfigFile != null) { AddTableCommand addTableCommand = new AddTableCommand().setControllerPort(_controllerPort).setFilePath(_tableConfigFile).setExecute(true); addTableCommand.execute(); return; } if (_tableName == null) { LOGGER.error("Table info not specified in configuration, please specify either config file or table name"); return; } String controllerAddress = "http://" + _localhost + ":" + _controllerPort; String tableJSONConfigString = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(_tableName) .setTimeColumnName(_timeColumnName).setTimeType(_timeUnit).setNumReplicas(3).setBrokerTenant("broker") .setServerTenant("server").build().toJSONConfigString(); sendPostRequest(ControllerRequestURLBuilder.baseUrl(controllerAddress).forTableCreate(), tableJSONConfigString); }
@Test public void testOfflineTableWithNullQuotaButWithRealtimeTableConfigNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); TableConfig realtimeTableConfig = new TableConfig.Builder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setRealtimeTableConfig(_testPropertyStore, REALTIME_TABLE_NAME, TableConfig.toZnRecord(realtimeTableConfig)); ExternalView brokerResource = generateBrokerResource(OFFLINE_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(OFFLINE_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); // Nothing happened since it doesn't have qps quota. _tableQueryQuotaManager.dropTableQueryQuota(OFFLINE_TABLE_NAME); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
private TableConfig buildOfflineTableConfig() throws Exception { // Create the replica group aware assignment strategy config ReplicaGroupStrategyConfig replicaGroupStrategyConfig = new ReplicaGroupStrategyConfig(); replicaGroupStrategyConfig.setNumInstancesPerPartition(NUM_PARTITION); replicaGroupStrategyConfig.setMirrorAssignmentAcrossReplicaGroups(true); // Create the routing config RoutingConfig routingConfig = new RoutingConfig(); routingConfig.setRoutingTableBuilderName("PartitionAwareOffline"); // Create table config TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(OFFLINE_TABLE_NAME) .setNumReplicas(NUM_REPLICA).setSegmentAssignmentStrategy("ReplicaGroupSegmentAssignmentStrategy").build(); tableConfig.getValidationConfig().setReplicaGroupStrategyConfig(replicaGroupStrategyConfig); tableConfig.setRoutingConfig(routingConfig); return tableConfig; }
@Test public void testRealtimeTableWithNullQuotaButWithOfflineTableConfigNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); TableConfig offlineTableConfig = new TableConfig.Builder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setOfflineTableConfig(_testPropertyStore, OFFLINE_TABLE_NAME, TableConfig.toZnRecord(offlineTableConfig)); ExternalView brokerResource = generateBrokerResource(REALTIME_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(REALTIME_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
@Test public void testRealtimeTableWithNullQuotaButWithOfflineTableConfigNotNullQpsConfig() throws Exception { QuotaConfig quotaConfig = new QuotaConfig(); quotaConfig.setStorage("6G"); quotaConfig.setMaxQueriesPerSecond("100.00"); TableConfig offlineTableConfig = new TableConfig.Builder(TableType.OFFLINE).setTableName(RAW_TABLE_NAME).setQuotaConfig(quotaConfig) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("1").setSegmentPushType("APPEND") .setBrokerTenant("testBroker").setServerTenant("testServer").build(); ZKMetadataProvider .setOfflineTableConfig(_testPropertyStore, OFFLINE_TABLE_NAME, TableConfig.toZnRecord(offlineTableConfig)); ExternalView brokerResource = generateBrokerResource(OFFLINE_TABLE_NAME); TableConfig tableConfig = generateDefaultTableConfig(REALTIME_TABLE_NAME); _tableQueryQuotaManager.initTableQueryQuota(tableConfig, brokerResource); Assert.assertEquals(_tableQueryQuotaManager.getRateLimiterMapSize(), 0); }
@BeforeClass public void setUp() throws Exception { startZk(); startController(); ControllerRequestBuilderUtil .addFakeBrokerInstancesToAutoJoinHelixCluster(getHelixClusterName(), ZkStarter.DEFAULT_ZK_STR, 5, true); ControllerRequestBuilderUtil .addFakeDataInstancesToAutoJoinHelixCluster(getHelixClusterName(), ZkStarter.DEFAULT_ZK_STR, 5, true); Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(getHelixClusterName(), "DefaultTenant_BROKER").size(), 5); // Adding table TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(TABLE_NAME) .setSegmentAssignmentStrategy("RandomAssignmentStrategy").setNumReplicas(2).build(); _helixResourceManager.addTable(tableConfig); }
private TableConfig createRealtimeTableConfig1(int replicaCount) { return new TableConfig.Builder(CommonConstants.Helix.TableType.REALTIME).setTableName(TEST_TABLE_NAME).setLLC(true) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("5").setNumReplicas(replicaCount).build(); }
private TableConfig generateDefaultTableConfig(String tableName) { TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName); TableConfig.Builder builder = new TableConfig.Builder(tableType); builder.setTableName(tableName); return builder.build(); }
private TableConfig generateTableConfig(String tableName) { TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName); Builder builder = new TableConfig.Builder(tableType); builder.setTableName(tableName); return builder.build(); } }
private void addingTableToPropertyStore(String tableName) throws Exception { TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(tableName) .setTimeColumnName("timestamp").setTimeType("DAYS").build(); ZKMetadataProvider .setOfflineTableConfig(_propertyStore, tableConfig.getTableName(), TableConfig.toZnRecord(tableConfig)); } }
private static TableConfig getOfflineTableConfig(String tableName, String timeColumnName, String timeType, String brokerTenant, String serverTenant, String loadMode, SegmentVersion segmentVersion, List<String> invertedIndexColumns, List<String> bloomFilterColumns, TableTaskConfig taskConfig) throws Exception { return new TableConfig.Builder(Helix.TableType.OFFLINE).setTableName(tableName).setTimeColumnName(timeColumnName) .setTimeType(timeType).setNumReplicas(3).setBrokerTenant(brokerTenant).setServerTenant(serverTenant) .setLoadMode(loadMode).setSegmentVersion(segmentVersion.toString()) .setInvertedIndexColumns(invertedIndexColumns).setBloomFilterColumns(bloomFilterColumns) .setTaskConfig(taskConfig).build(); }
private TableConfig generateTableConfig(String tableName) throws Exception { TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName); Builder builder = new TableConfig.Builder(tableType); builder.setTableName(tableName); return builder.build(); } }
public void configureTable(String tableName, List<String> invertedIndexColumns, List<String> bloomFilterColumns) throws Exception { TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(tableName) .setSegmentAssignmentStrategy(_segmentAssignmentStrategy).setNumReplicas(_numReplicas) .setBrokerTenant(_brokerTenantName).setServerTenant(_serverTenantName).setLoadMode(_loadMode) .setSegmentVersion(_segmentFormatVersion).setInvertedIndexColumns(invertedIndexColumns) .setBloomFilterColumns(bloomFilterColumns).build(); _helixResourceManager.addTable(tableConfig); }
private TableConfig createOfflineTableConfig() throws Exception { return new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(TEST_TABLE_NAME) .setRetentionTimeUnit("DAYS").setRetentionTimeValue("365").setNumReplicas(2).build(); }