public static IdealState buildInitialHighLevelRealtimeIdealStateFor(String realtimeTableName, TableConfig realtimeTableConfig, HelixManager helixManager, ZkHelixPropertyStore<ZNRecord> zkHelixPropertyStore, boolean enableBatchMessageMode) { RealtimeTagConfig realtimeTagConfig = new RealtimeTagConfig(realtimeTableConfig); final List<String> realtimeInstances = HelixHelper.getInstancesWithTag(helixManager, realtimeTagConfig.getConsumingServerTag()); IdealState idealState = buildEmptyRealtimeIdealStateFor(realtimeTableName, 1, enableBatchMessageMode); if (realtimeInstances.size() % Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()) != 0) { throw new RuntimeException( "Number of instance in current tenant should be an integer multiples of the number of replications"); } setupInstanceConfigForHighLevelConsumer(realtimeTableName, realtimeInstances.size(), Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()), realtimeTableConfig.getIndexingConfig().getStreamConfigs(), zkHelixPropertyStore, realtimeInstances); return idealState; }
/** * If realtime table and includeConsuming=true, rebalance consuming segments. NewPartitionAssignment will be used only * in this case. Always rebalance completed (online) segments, NewPartitionAssignment unused in this case * @param idealState old ideal state * @param tableConfig table config of table tor rebalance * @param rebalanceUserConfig custom user configs for specific rebalance strategies * @param newPartitionAssignment new rebalanced partition assignments as part of the resource rebalance * @return */ @Override public IdealState getRebalancedIdealState(IdealState idealState, TableConfig tableConfig, Configuration rebalanceUserConfig, PartitionAssignment newPartitionAssignment) { String tableNameWithType = tableConfig.getTableName(); CommonConstants.Helix.TableType tableType = tableConfig.getTableType(); LOGGER.info("Rebalancing ideal state for table {}", tableNameWithType); // get target num replicas int targetNumReplicas; if (tableType.equals(CommonConstants.Helix.TableType.REALTIME)) { String replicasString = tableConfig.getValidationConfig().getReplicasPerPartition(); try { targetNumReplicas = Integer.parseInt(replicasString); } catch (Exception e) { throw new RuntimeException("Invalid value for replicasPerPartition:'" + replicasString + "'", e); } } else { targetNumReplicas = Integer.parseInt(tableConfig.getValidationConfig().getReplication()); } return rebalanceIdealState(idealState, tableConfig, targetNumReplicas, rebalanceUserConfig, newPartitionAssignment); }
final int newNumReplicas = Integer.parseInt(offlineTableConfig.getValidationConfig().getReplication());
int numReplicasInIdealState = Integer.parseInt(currentIdealState.getReplicas()); final TableConfig offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(propertyStore, tableName); final int numReplicasInTableConfig = Integer.parseInt(offlineTableConfig.getValidationConfig().getReplication());
public void setExistingTableConfig(TableConfig config, String tableNameWithType, TableType type) throws IOException { if (type == TableType.REALTIME) { ZKMetadataProvider.setRealtimeTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(config)); ensureRealtimeClusterIsSetUp(config, tableNameWithType, config.getIndexingConfig()); } else if (type == TableType.OFFLINE) { // Update replica group partition assignment to the property store if applicable updateReplicaGroupPartitionAssignment(config); ZKMetadataProvider.setOfflineTableConfig(_propertyStore, tableNameWithType, TableConfig.toZnRecord(config)); IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableNameWithType); final String configReplication = config.getValidationConfig().getReplication(); if (configReplication != null && !config.getValidationConfig().getReplication() .equals(idealState.getReplicas())) { HelixHelper.updateIdealState(_helixZkManager, tableNameWithType, new Function<IdealState, IdealState>() { @Nullable @Override public IdealState apply(@Nullable IdealState idealState) { idealState.setReplicas(configReplication); return idealState; } }, RetryPolicies.exponentialBackoffRetryPolicy(5, 1000L, 1.2f)); } } }
/** * Helper method to add the passed in offline segment to the helix cluster. * - Gets the segment name and the table name from the passed in segment meta-data. * - Identifies the instance set onto which the segment needs to be added, based on * segment assignment strategy and replicas in the table config in the property-store. * - Updates ideal state such that the new segment is assigned to required set of instances as per * the segment assignment strategy and replicas. * * @param segmentMetadata Meta-data for the segment, used to access segmentName and tableName. */ // NOTE: method should be thread-safe private void addNewOfflineSegment(SegmentMetadata segmentMetadata) { String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(segmentMetadata.getTableName()); String segmentName = segmentMetadata.getName(); // Assign new segment to instances TableConfig offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, offlineTableName); Preconditions.checkNotNull(offlineTableConfig); int numReplicas = Integer.parseInt(offlineTableConfig.getValidationConfig().getReplication()); String serverTenant = TagNameUtils.getOfflineTagForTenant(offlineTableConfig.getTenantConfig().getServer()); SegmentAssignmentStrategy segmentAssignmentStrategy = SegmentAssignmentStrategyFactory .getSegmentAssignmentStrategy(offlineTableConfig.getValidationConfig().getSegmentAssignmentStrategy()); List<String> assignedInstances = segmentAssignmentStrategy .getAssignedInstances(_helixZkManager, _helixAdmin, _propertyStore, _helixClusterName, segmentMetadata, numReplicas, serverTenant); HelixHelper.addSegmentToIdealState(_helixZkManager, offlineTableName, segmentName, assignedInstances); }
.buildEmptyIdealStateFor(tableNameWithType, Integer.parseInt(segmentsConfig.getReplication()), _enableBatchMessageMode); LOGGER.info("adding table via the admin");