@Override public void init(Configuration configuration, TableConfig tableConfig, ZkHelixPropertyStore<ZNRecord> propertyStore, BrokerMetrics brokerMetrics) { super.init(configuration, tableConfig, propertyStore, brokerMetrics); _numReplicas = Integer.valueOf(tableConfig.getValidationConfig().getReplicasPerPartition()); }
@Override public void init(Configuration configuration, TableConfig tableConfig, ZkHelixPropertyStore<ZNRecord> propertyStore, BrokerMetrics brokerMetrics) { super.init(configuration, tableConfig, propertyStore, brokerMetrics); String partitionColumn = tableConfig.getValidationConfig().getReplicaGroupStrategyConfig().getPartitionColumn(); _isPartitionLevelReplicaGroupAssignment = (partitionColumn != null); _numReplicas = tableConfig.getValidationConfig().getReplicationNumber(); }
private boolean shouldSendTimeboundaryRefreshMsg(String rawTableName, TableConfig tableConfig) { if (!hasOfflineTable(rawTableName) || !hasRealtimeTable(rawTableName)) { return false; } if (tableConfig == null) { return false; } SegmentsValidationAndRetentionConfig validationConfig = tableConfig.getValidationConfig(); return validationConfig != null && APPEND.equals(validationConfig.getSegmentPushType()); }
public static IdealState buildInitialHighLevelRealtimeIdealStateFor(String realtimeTableName, TableConfig realtimeTableConfig, HelixManager helixManager, ZkHelixPropertyStore<ZNRecord> zkHelixPropertyStore, boolean enableBatchMessageMode) { RealtimeTagConfig realtimeTagConfig = new RealtimeTagConfig(realtimeTableConfig); final List<String> realtimeInstances = HelixHelper.getInstancesWithTag(helixManager, realtimeTagConfig.getConsumingServerTag()); IdealState idealState = buildEmptyRealtimeIdealStateFor(realtimeTableName, 1, enableBatchMessageMode); if (realtimeInstances.size() % Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()) != 0) { throw new RuntimeException( "Number of instance in current tenant should be an integer multiples of the number of replications"); } setupInstanceConfigForHighLevelConsumer(realtimeTableName, realtimeInstances.size(), Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()), realtimeTableConfig.getIndexingConfig().getStreamConfigs(), zkHelixPropertyStore, realtimeInstances); return idealState; }
public static void buildLowLevelRealtimeIdealStateFor(String realtimeTableName, TableConfig realtimeTableConfig, IdealState idealState, boolean enableBatchMessageMode) { // Validate replicasPerPartition here. final String replicasPerPartitionStr = realtimeTableConfig.getValidationConfig().getReplicasPerPartition(); if (replicasPerPartitionStr == null || replicasPerPartitionStr.isEmpty()) { throw new RuntimeException("Null or empty value for replicasPerPartition, expected a number"); } final int nReplicas; try { nReplicas = Integer.valueOf(replicasPerPartitionStr); } catch (NumberFormatException e) { throw new PinotHelixResourceManager.InvalidTableConfigException( "Invalid value for replicasPerPartition, expected a number: " + replicasPerPartitionStr, e); } if (idealState == null) { idealState = buildEmptyRealtimeIdealStateFor(realtimeTableName, nReplicas, enableBatchMessageMode); } final PinotLLCRealtimeSegmentManager segmentManager = PinotLLCRealtimeSegmentManager.getInstance(); try { segmentManager.setupNewTable(realtimeTableConfig, idealState); } catch (InvalidConfigException e) { throw new IllegalStateException("Caught exception when creating table " + realtimeTableName, e); } }
/** * Generates stream partition assignment for given table, using tagged hosts and num partitions */ public PartitionAssignment generateStreamPartitionAssignment(TableConfig tableConfig, int numPartitions) throws InvalidConfigException { // TODO: add an override which can read from znode, instead of generating on the fly List<String> partitions = new ArrayList<>(numPartitions); for (int i = 0; i < numPartitions; i++) { partitions.add(String.valueOf(i)); } String tableNameWithType = tableConfig.getTableName(); int numReplicas = tableConfig.getValidationConfig().getReplicasPerPartitionNumber(); List<String> consumingTaggedInstances = getConsumingTaggedInstances(tableConfig); if (consumingTaggedInstances.size() < numReplicas) { throw new InvalidConfigException( "Not enough consuming instances tagged. Must be atleast equal to numReplicas:" + numReplicas); } /** * TODO: We will use only uniform assignment for now * This will be refactored as AssignmentStrategy interface and implementations UniformAssignment, BalancedAssignment etc * {@link StreamPartitionAssignmentGenerator} and AssignmentStrategy interface will together replace * StreamPartitionAssignmentGenerator and StreamPartitionAssignmentStrategy */ return uniformAssignment(tableNameWithType, partitions, numReplicas, consumingTaggedInstances); }
private String getDefaultSegmentName(TableConfig tableConfig, Schema schema, List<File> inputIndexDirs, long minStartTime, long maxEndTime) throws Exception { String tableName = tableConfig.getTableName(); // Fetch time related configurations from schema and table config. String pushFrequency = tableConfig.getValidationConfig().getSegmentPushFrequency(); String timeColumnType = tableConfig.getValidationConfig().getTimeType(); String pushType = tableConfig.getValidationConfig().getSegmentPushType(); String timeFormat = schema.getTimeFieldSpec().getOutgoingGranularitySpec().getTimeFormat(); // Generate the final segment name using segment name generator NormalizedDateSegmentNameGenerator segmentNameGenerator = new NormalizedDateSegmentNameGenerator(tableName, DEFAULT_SEQUENCE_ID, timeColumnType, pushFrequency, pushType, null, null, timeFormat); return segmentNameGenerator.generateSegmentName(minStartTime, maxEndTime); }
@BeforeClass public void setUp() { _tableSizeReader = mock(TableSizeReader.class); _tableConfig = mock(TableConfig.class); _quotaConfig = mock(QuotaConfig.class); _controllerMetrics = new ControllerMetrics(new MetricsRegistry()); _validationConfig = mock(SegmentsValidationAndRetentionConfig.class); _pinotHelixResourceManager = mock(PinotHelixResourceManager.class); when(_tableConfig.getValidationConfig()).thenReturn(_validationConfig); when(_validationConfig.getReplicationNumber()).thenReturn(2); TEST_DIR.mkdirs(); }
private void deleteSchemaInternal(String schemaName) { Schema schema = _pinotHelixResourceManager.getSchema(schemaName); if (schema == null) { throw new ControllerApplicationException(LOGGER, String.format("Schema %s not found", schemaName), Response.Status.NOT_FOUND); } // If the schema is associated with a table, we should not delete it. List<String> tableNames = _pinotHelixResourceManager.getAllRealtimeTables(); for (String tableName : tableNames) { TableConfig config = _pinotHelixResourceManager.getRealtimeTableConfig(tableName); String tableSchema = config.getValidationConfig().getSchemaName(); if (schemaName.equals(tableSchema)) { throw new ControllerApplicationException(LOGGER, String.format("Cannot delete schema %s, as it is associated with table %s", schemaName, tableName), Response.Status.CONFLICT); } } LOGGER.info("Trying to delete schema {}", schemaName); if (_pinotHelixResourceManager.deleteSchema(schema)) { LOGGER.info("Success: Deleted schema {}", schemaName); } else { throw new ControllerApplicationException(LOGGER, String.format("Failed to delete schema %s", schemaName), Response.Status.INTERNAL_SERVER_ERROR); } } }
private void checkTableConfigWithAssignmentConfig(TableConfig tableConfig, TableConfig tableConfigToCompare) { // Check that the segment assignment configuration does exist. Assert.assertEquals(tableConfigToCompare.getTableName(), tableConfig.getTableName()); Assert.assertNotNull(tableConfigToCompare.getValidationConfig().getReplicaGroupStrategyConfig()); Assert.assertEquals(tableConfigToCompare.getValidationConfig().getReplicaGroupStrategyConfig(), tableConfig.getValidationConfig().getReplicaGroupStrategyConfig()); // Check that the configurations are correct. ReplicaGroupStrategyConfig strategyConfig = tableConfigToCompare.getValidationConfig().getReplicaGroupStrategyConfig(); Assert.assertTrue(strategyConfig.getMirrorAssignmentAcrossReplicaGroups()); Assert.assertEquals(strategyConfig.getNumInstancesPerPartition(), 5); Assert.assertEquals(strategyConfig.getPartitionColumn(), "memberId"); }
private void checkTableConfigWithHllConfig(TableConfig tableConfig, TableConfig tableConfigToCompare) { // Check that the segment assignment configuration does exist. Assert.assertEquals(tableConfigToCompare.getTableName(), tableConfig.getTableName()); Assert.assertNotNull(tableConfigToCompare.getValidationConfig().getHllConfig()); // Check that the configurations are correct. HllConfig hllConfig = tableConfigToCompare.getValidationConfig().getHllConfig(); Set<String> columns = new HashSet<>(); columns.add("column"); columns.add("column2"); Assert.assertEquals(hllConfig.getColumnsToDeriveHllFields(), columns); Assert.assertEquals(hllConfig.getHllLog2m(), 9); Assert.assertEquals(hllConfig.getHllDeriveColumnSuffix(), "suffix"); } }
public RebalanceSegmentStrategy getRebalanceSegmentsStrategy(TableConfig tableConfig) { // If we use replica group segment assignment strategy, we pick the replica group rebalancer String segmentAssignmentStrategy = tableConfig.getValidationConfig().getSegmentAssignmentStrategy(); if (segmentAssignmentStrategy == null) { return new DefaultRebalanceSegmentStrategy(_helixManager); } switch (SegmentAssignmentStrategyEnum.valueOf(segmentAssignmentStrategy)) { case ReplicaGroupSegmentAssignmentStrategy: return new ReplicaGroupRebalanceSegmentStrategy(_helixManager); default: return new DefaultRebalanceSegmentStrategy(_helixManager); } } }
private void updateTableConfig(int targetNumInstancePerPartition, int targetNumReplicaGroup) throws IOException { String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME); TableConfig tableConfig = _helixResourceManager.getTableConfig(TABLE_NAME, CommonConstants.Helix.TableType.OFFLINE); tableConfig.getValidationConfig().getReplicaGroupStrategyConfig() .setNumInstancesPerPartition(targetNumInstancePerPartition); tableConfig.getValidationConfig().setReplication(Integer.toString(targetNumReplicaGroup)); _helixResourceManager .setExistingTableConfig(tableConfig, tableNameWithType, CommonConstants.Helix.TableType.OFFLINE); } }
private void validateLoadedConfig(TableConfig config) { Assert.assertEquals(config.getTableName(), "mytable_OFFLINE"); Assert.assertEquals(config.getQuotaConfig().getStorage(), "125 GiB"); Assert.assertEquals(config.getValidationConfig().getRetentionTimeValue(), "5"); Assert.assertEquals(config.getValidationConfig().getRetentionTimeUnit(), "DAYS"); Assert.assertEquals(config.getTenantConfig().getBroker(), "foo"); Assert.assertEquals(config.getTenantConfig().getServer(), "bar"); Assert.assertEquals(config.getIndexingConfig().getSortedColumn(), Lists.newArrayList("foo")); } }
/** * Helper method to add the passed in offline segment to the helix cluster. * - Gets the segment name and the table name from the passed in segment meta-data. * - Identifies the instance set onto which the segment needs to be added, based on * segment assignment strategy and replicas in the table config in the property-store. * - Updates ideal state such that the new segment is assigned to required set of instances as per * the segment assignment strategy and replicas. * * @param segmentMetadata Meta-data for the segment, used to access segmentName and tableName. */ // NOTE: method should be thread-safe private void addNewOfflineSegment(SegmentMetadata segmentMetadata) { String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(segmentMetadata.getTableName()); String segmentName = segmentMetadata.getName(); // Assign new segment to instances TableConfig offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, offlineTableName); Preconditions.checkNotNull(offlineTableConfig); int numReplicas = Integer.parseInt(offlineTableConfig.getValidationConfig().getReplication()); String serverTenant = TagNameUtils.getOfflineTagForTenant(offlineTableConfig.getTenantConfig().getServer()); SegmentAssignmentStrategy segmentAssignmentStrategy = SegmentAssignmentStrategyFactory .getSegmentAssignmentStrategy(offlineTableConfig.getValidationConfig().getSegmentAssignmentStrategy()); List<String> assignedInstances = segmentAssignmentStrategy .getAssignedInstances(_helixZkManager, _helixAdmin, _propertyStore, _helixClusterName, segmentMetadata, numReplicas, serverTenant); HelixHelper.addSegmentToIdealState(_helixZkManager, offlineTableName, segmentName, assignedInstances); }
public void setExistingTableConfig(TableConfig config, String tableNameWithType, TableType type) throws IOException { if (type == TableType.REALTIME) { ZKMetadataProvider.setRealtimeTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(config)); ensureRealtimeClusterIsSetUp(config, tableNameWithType, config.getIndexingConfig()); } else if (type == TableType.OFFLINE) { // Update replica group partition assignment to the property store if applicable updateReplicaGroupPartitionAssignment(config); ZKMetadataProvider.setOfflineTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(config)); IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableNameWithType); final String configReplication = config.getValidationConfig().getReplication(); if (configReplication != null && !config.getValidationConfig().getReplication() .equals(idealState.getReplicas())) { HelixHelper.updateIdealState(_helixZkManager, tableNameWithType, new Function<IdealState, IdealState>() { @Nullable @Override public IdealState apply(@Nullable IdealState idealState) { idealState.setReplicas(configReplication); return idealState; } }, RetryPolicies.exponentialBackoffRetryPolicy(5, 1000L, 1.2f)); } } }
/** * Update replica group partition assignment in the property store * * @param tableConfig a table config */ private void updateReplicaGroupPartitionAssignment(TableConfig tableConfig) { String tableNameWithType = tableConfig.getTableName(); String assignmentStrategy = tableConfig.getValidationConfig().getSegmentAssignmentStrategy(); // We create replica group partition assignment and write to property store if new table config // has the replica group config. if (assignmentStrategy != null && SegmentAssignmentStrategyEnum.valueOf(assignmentStrategy) == SegmentAssignmentStrategyEnum.ReplicaGroupSegmentAssignmentStrategy) { ReplicaGroupPartitionAssignmentGenerator partitionAssignmentGenerator = new ReplicaGroupPartitionAssignmentGenerator(_propertyStore); // Create the new replica group partition assignment if there is none in the property store. // This will create the replica group partition assignment and write to the property store in 2 cases: // 1. when we create the table with replica group segment assignment // 2. when we update the table config with replica group segment assignment from another assignment strategy if (partitionAssignmentGenerator.getReplicaGroupPartitionAssignment(tableNameWithType) == null) { List<String> servers = getServerInstancesForTable(tableNameWithType, TableType.OFFLINE); ReplicaGroupPartitionAssignment partitionAssignment = partitionAssignmentGenerator.buildReplicaGroupPartitionAssignment(tableNameWithType, tableConfig, servers); partitionAssignmentGenerator.writeReplicaGroupPartitionAssignment(partitionAssignment); } } }
private void testTableMinReplicationInternal(String tableName, int tableReplication) throws Exception { String tableJSONConfigString = _offlineBuilder.setTableName(tableName).setNumReplicas(tableReplication).build().toJSONConfigString(); sendPostRequest(_createTableUrl, tableJSONConfigString); // table creation should succeed TableConfig tableConfig = getTableConfig(tableName, "OFFLINE"); Assert.assertEquals(tableConfig.getValidationConfig().getReplicationNumber(), Math.max(tableReplication, MIN_NUM_REPLICAS)); addDummySchema(tableName); tableJSONConfigString = _realtimeBuilder.setTableName(tableName).setNumReplicas(tableReplication).build().toJSONConfigString(); sendPostRequest(_createTableUrl, tableJSONConfigString); tableConfig = getTableConfig(tableName, "REALTIME"); Assert.assertEquals(tableConfig.getValidationConfig().getReplicationNumber(), Math.max(tableReplication, MIN_NUM_REPLICAS)); // This test can only be done via integration test // int replicasPerPartition = Integer.valueOf(tableConfig.getValidationConfig().getReplicasPerPartition()); // Assert.assertEquals(replicasPerPartition, Math.max(tableReplication, TABLE_MIN_REPLICATION)); }
private TableConfig buildRealtimeTableConfig() throws Exception { // Create partition config Map<String, ColumnPartitionConfig> metadataMap = new HashMap<>(); metadataMap.put(PARTITION_COLUMN, new ColumnPartitionConfig(PARTITION_FUNCTION_NAME, NUM_PARTITION)); SegmentPartitionConfig partitionConfig = new SegmentPartitionConfig(metadataMap); // Create the routing config RoutingConfig routingConfig = new RoutingConfig(); routingConfig.setRoutingTableBuilderName("PartitionAwareOffline"); // Create table config TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.REALTIME).setTableName(REALTIME_TABLE_NAME) .setNumReplicas(NUM_REPLICA).build(); tableConfig.getValidationConfig().setReplicasPerPartition(Integer.toString(NUM_REPLICA)); tableConfig.getIndexingConfig().setSegmentPartitionConfig(partitionConfig); tableConfig.setRoutingConfig(routingConfig); return tableConfig; }
private TableConfig buildOfflineTableConfig() throws Exception { // Create the replica group aware assignment strategy config ReplicaGroupStrategyConfig replicaGroupStrategyConfig = new ReplicaGroupStrategyConfig(); replicaGroupStrategyConfig.setNumInstancesPerPartition(NUM_PARTITION); replicaGroupStrategyConfig.setMirrorAssignmentAcrossReplicaGroups(true); // Create the routing config RoutingConfig routingConfig = new RoutingConfig(); routingConfig.setRoutingTableBuilderName("PartitionAwareOffline"); // Create table config TableConfig tableConfig = new TableConfig.Builder(CommonConstants.Helix.TableType.OFFLINE).setTableName(OFFLINE_TABLE_NAME) .setNumReplicas(NUM_REPLICA).setSegmentAssignmentStrategy("ReplicaGroupSegmentAssignmentStrategy").build(); tableConfig.getValidationConfig().setReplicaGroupStrategyConfig(replicaGroupStrategyConfig); tableConfig.setRoutingConfig(routingConfig); return tableConfig; }