/** * Returns a map from server instance to list of segments it serves for the given table. */ public Map<String, List<String>> getServerToSegmentsMap(String tableNameWithType) { Map<String, List<String>> serverToSegmentsMap = new HashMap<>(); IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableNameWithType); if (idealState == null) { throw new IllegalStateException("Ideal state does not exist for table: " + tableNameWithType); } for (String segment : idealState.getPartitionSet()) { for (String server : idealState.getInstanceStateMap(segment).keySet()) { serverToSegmentsMap.computeIfAbsent(server, key -> new ArrayList<>()).add(segment); } } return serverToSegmentsMap; }
/** * Returns all instances for the given resource. * * @param idealState IdealState of the resource for which to return the instances of. * @return Returns a Set of strings containing the instance names for the given cluster. */ public static Set<String> getAllInstancesForResource(IdealState idealState) { final Set<String> instances = new HashSet<String>(); for (final String partition : idealState.getPartitionSet()) { for (final String instance : idealState.getInstanceSet(partition)) { instances.add(instance); } } return instances; }
@Nullable @Override public IdealState apply(@Nullable IdealState input) { Map<String, Map<String, String>> existingMapField = input.getRecord().getMapFields(); for (Map.Entry<String, Map<String, String>> segmentEntry : proposedIdealState.entrySet()) { existingMapField.put(segmentEntry.getKey(), segmentEntry.getValue()); } return input; } }, RetryPolicies.exponentialBackoffRetryPolicy(5, 500L, 2.0f));
public static IdealState addNewRealtimeSegmentToIdealState(String segmentId, IdealState state, String instanceName) { state.setPartitionState(segmentId, instanceName, ONLINE); state.setNumPartitions(state.getNumPartitions() + 1); return state; }
/** * Sets the idealstate for a given segment to the target mapping */ private void setTargetState(IdealState idealState, String segmentId, Map<String, String> targetMap) { idealState.getInstanceStateMap(segmentId).clear(); idealState.setInstanceStateMap(segmentId, targetMap); }
List<String> allTableNames = new ArrayList<String>(); allTableNames.add(tableName); IdealState idealState = new IdealState(tableName); idealState.setPartitionState("myTable_0", "pinot1", "ONLINE"); idealState.setPartitionState("myTable_0", "pinot2", "ONLINE"); idealState.setPartitionState("myTable_0", "pinot3", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot1", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot2", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot3", "ONLINE"); idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE"); idealState.setReplicas("2"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
public static IdealState resetCustomIdealStateFor(IdealState oldIdealState, String topicName, String partitionToReplace, String newInstanceName) { final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName); int oldNumPartitions = oldIdealState.getNumPartitions(); customModeIdealStateBuilder .setStateModel(OnlineOfflineStateModel.name) .setNumPartitions(oldNumPartitions).setNumReplica(1) .setMaxPartitionsPerNode(oldNumPartitions); for (String partitionName : oldIdealState.getPartitionSet()) { String instanceName = oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next(); String instanceToUse = partitionName.equals(partitionToReplace) ? newInstanceName : instanceName; customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceToUse, "ONLINE"); } return customModeIdealStateBuilder.build(); }
@Test public void testReplicas() { IdealState idealState = new IdealState("test-db"); idealState.setRebalanceMode(RebalanceMode.SEMI_AUTO); idealState.setNumPartitions(4); idealState.setStateModelDefRef("MasterSlave"); idealState.setReplicas("" + 2); List<String> preferenceList = new ArrayList<String>(); preferenceList.add("node_0"); idealState.getRecord().setListField("test-db_0", preferenceList); Assert.assertFalse(idealState.isValid(), "should fail since replicas not equals to preference-list size"); preferenceList.add("node_1"); idealState.getRecord().setListField("test-db_0", preferenceList); Assert.assertTrue(idealState.isValid(), "should pass since replicas equals to preference-list size"); }
@Override public void addResource(String clusterName, String resourceName, int partitions, String stateModelRef, String rebalancerMode, String rebalanceStrategy, int bucketSize, int maxPartitionsPerInstance) { if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { throw new HelixException("cluster " + clusterName + " is not setup yet"); } IdealState idealState = new IdealState(resourceName); idealState.setNumPartitions(partitions); idealState.setStateModelDefRef(stateModelRef); RebalanceMode mode = idealState.rebalanceModeFromString(rebalancerMode, RebalanceMode.SEMI_AUTO); idealState.setRebalanceMode(mode); idealState.setRebalanceStrategy(rebalanceStrategy); idealState.setReplicas("" + 0); idealState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY); if (maxPartitionsPerInstance > 0 && maxPartitionsPerInstance < Integer.MAX_VALUE) { idealState.setMaxPartitionsPerInstance(maxPartitionsPerInstance); } if (bucketSize > 0) { idealState.setBucketSize(bucketSize); } addResource(clusterName, resourceName, idealState); }
/** * Create an IdealState for a resource that belongs to a resource group We use * "resourceGroupName$resourceInstanceTag" as the IdealState znode name to differetiate different * resources from the same resourceGroup. */ public IdealState createIdealStateForResourceGroup(String resourceGroupName, String resourceTag, int numPartition, int replica, String rebalanceMode, String stateModelDefName) { String idealStateId = genIdealStateNameWithResourceTag(resourceGroupName, resourceTag); IdealState idealState = new IdealState(idealStateId); idealState.setNumPartitions(numPartition); idealState.setStateModelDefRef(stateModelDefName); IdealState.RebalanceMode mode = idealState.rebalanceModeFromString(rebalanceMode, IdealState.RebalanceMode.SEMI_AUTO); idealState.setRebalanceMode(mode); idealState.setReplicas("" + replica); idealState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY); idealState.setResourceGroupName(resourceGroupName); idealState.setInstanceGroupTag(resourceTag); idealState.enableGroupRouting(true); return idealState; }
Assert.assertEquals( _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE) .getPartitionSet().size(), 1); Assert.assertEquals( _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE) .getInstanceSet(tableName + "_OFFLINE").size(), 20); .assertEquals(_helixAdmin.getResourceIdealState(_helixClusterName, tableName + "_OFFLINE").getNumPartitions(), i); _helixResourceManager.addNewSegment(SegmentMetadataMockUtils.mockSegmentMetadata(tableName), "downloadUrl"); Assert .assertEquals(_helixAdmin.getResourceIdealState(_helixClusterName, tableName + "_OFFLINE").getNumPartitions(), i + 1); Assert.assertEquals( _helixAdmin.getResourceIdealState(_helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE) .getPartitionSet().size(), 0);
private Set<String> fetchLatestTableResources(HelixAdmin helixAdmin) { Set<String> resourcesToMonitor = new HashSet<>(); for (String resourceName : helixAdmin.getResourcesInCluster(_helixClusterName)) { // Only monitor table resources if (!TableNameBuilder.isTableResource(resourceName)) { continue; } // Only monitor enabled resources IdealState idealState = helixAdmin.getResourceIdealState(_helixClusterName, resourceName); if (idealState.isEnabled()) { for (String partitionName : idealState.getPartitionSet()) { if (idealState.getInstanceSet(partitionName).contains(_instanceId)) { resourcesToMonitor.add(resourceName); break; } } } } return resourcesToMonitor; }
@Override public void addClusterToGrandCluster(String clusterName, String grandCluster) { logger.info("Add cluster {} to grand cluster {}.", clusterName, grandCluster); if (!ZKUtil.isClusterSetup(grandCluster, _zkClient)) { throw new HelixException("Grand cluster " + grandCluster + " is not setup yet"); } if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { throw new HelixException("Cluster " + clusterName + " is not setup yet"); } IdealState idealState = new IdealState(clusterName); idealState.setNumPartitions(1); idealState.setStateModelDefRef("LeaderStandby"); List<String> controllers = getInstancesInCluster(grandCluster); if (controllers.size() == 0) { throw new HelixException("Grand cluster " + grandCluster + " has no instances"); } idealState.setReplicas(Integer.toString(controllers.size())); Collections.shuffle(controllers); idealState.getRecord().setListField(clusterName, controllers); idealState.setPartitionState(clusterName, controllers.get(0), "LEADER"); for (int i = 1; i < controllers.size(); i++) { idealState.setPartitionState(clusterName, controllers.get(i), "STANDBY"); } ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(grandCluster, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); Builder keyBuilder = accessor.keyBuilder(); accessor.setProperty(keyBuilder.idealStates(idealState.getResourceName()), idealState); }
if (!idealState.isEnabled()) { if (_logDisabledTables) { LOGGER.warn("Table {} is disabled. Skipping segment status checks", tableNameWithType); if (idealState.getPartitionSet().isEmpty()) { int nReplicasFromIdealState = 1; try { nReplicasFromIdealState = Integer.valueOf(idealState.getReplicas()); } catch (NumberFormatException e) { .setValueOfTableGauge(tableNameWithType, ControllerGauge.IDEALSTATE_ZNODE_SIZE, idealState.toString().length()); _metricsRegistry.setValueOfTableGauge(tableNameWithType, ControllerGauge.SEGMENT_COUNT, (long) (idealState.getPartitionSet().size())); ExternalView externalView = _pinotHelixResourceManager.getTableExternalView(tableNameWithType); int nOffline = 0; // Keeps track of number segments with no online replicas int nSegments = 0; // Counts number of segments for (String partitionName : idealState.getPartitionSet()) { int nReplicas = 0; int nIdeal = 0; nSegments++; for (Map.Entry<String, String> serverAndState : idealState.getInstanceStateMap(partitionName).entrySet()) { if (serverAndState == null) { break; nReplicasIdealMax = (idealState.getInstanceStateMap(partitionName).size() > nReplicasIdealMax) ? idealState .getInstanceStateMap(partitionName).size() : nReplicasIdealMax; if ((externalView == null) || (externalView.getStateMap(partitionName) == null)) {
private IdealState updateIdealState(IdealState idealState, int newNumReplicas) { idealState.setReplicas(Integer.toString(newNumReplicas)); Set<String> segmentIds = idealState.getPartitionSet(); for (String segmentId : segmentIds) { Map<String, String> instanceStateMap = idealState.getInstanceStateMap(segmentId); if (instanceStateMap.size() > newNumReplicas) { Set<String> keys = instanceStateMap.keySet(); while (instanceStateMap.size() > newNumReplicas) { instanceStateMap.remove(keys.iterator().next()); } } else if (instanceStateMap.size() < newNumReplicas) { throw new RuntimeException( "Segment " + segmentId + " has " + instanceStateMap.size() + " replicas but want changed to " + newNumReplicas); } } return idealState; }
if (onlineSegments != null) { for (String segment : onlineSegments) { Set<String> oldInstances = idealState.getInstanceSet(segment); Preconditions.checkArgument(CollectionUtils.isNotEmpty(oldInstances)); for (String instance : oldInstances) { idealState .setPartitionState(segment, instance, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE); for (String segment : consumingSegments) { List<String> newInstances = assignments.get(segment); Map<String, String> instanceStateMap = idealState.getInstanceStateMap(segment); if (instanceStateMap != null) { instanceStateMap.clear(); .setPartitionState(segment, instance, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
@Override public IdealState apply(IdealState idealState) { if (idealState.getPartitionSet().contains(resourceTag)) { idealState.getPartitionSet().remove(resourceTag); return idealState; } else { return null; } } };
if (idealState == null || !idealState.isEnabled()) { iterator.remove(); continue; for (String partitionName : idealState.getPartitionSet()) { String idealStateStatus = idealState.getInstanceStateMap(partitionName).get(_instanceName);
instanceAssignments = strategy.assign(Lists.newArrayList(newSegmentId), partitionAssignment); } catch (InvalidConfigException e) { _controllerMetrics.addMeteredTableValue(idealState.getResourceName(), ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L); throw new IllegalStateException("Caught exception when updating ideal state on segment completion", e); Set<String> currentSegmentInstances = idealState.getInstanceSet(currentSegmentId); for (String instance : currentSegmentInstances) { idealState.setPartitionState(currentSegmentId, instance, PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE); Map<String, String> stateMap = idealState.getInstanceStateMap(newSegmentId); if (stateMap != null) { stateMap.clear(); .setPartitionState(newSegmentId, instance, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
@Override public IdealState apply(IdealState idealState) { Set<String> partitions = idealState.getPartitionSet(); if (partitions.contains(segmentName)) { LOGGER.warn("Segment already exists in the ideal state for segment: {} of table: {}, do not update", segmentName, tableNameWithType); } else { if (assignedInstances.isEmpty()) { LOGGER.warn("No instance assigned for segment: {} of table: {}", segmentName, tableNameWithType); } else { int numPartitions = partitions.size() + 1; for (String instance : assignedInstances) { idealState.setPartitionState(segmentName, instance, ONLINE); } idealState.setNumPartitions(numPartitions); } } return idealState; } };