@Nullable @Override public IdealState apply(@Nullable IdealState input) { Map<String, Map<String, String>> existingMapField = input.getRecord().getMapFields(); for (Map.Entry<String, Map<String, String>> segmentEntry : proposedIdealState.entrySet()) { existingMapField.put(segmentEntry.getKey(), segmentEntry.getValue()); } return input; } }, RetryPolicies.exponentialBackoffRetryPolicy(5, 500L, 2.0f));
public static SegmentMergeLineage fromZNRecord(ZNRecord record) { String tableNameWithType = record.getId(); Map<String, List<String>> segmentGroupLineageMap = record.getListFields(); Map<Integer, Map<String, List<String>>> groupToSegmentsMap = new HashMap<>(); for (Map.Entry<String, Map<String, String>> entry : record.getMapFields().entrySet()) { String levelKey = entry.getKey(); Integer level = Integer.parseInt(levelKey.substring(LEVEL_KEY_PREFIX.length())); Map<String, List<String>> groupToSegmentsForLevel = new HashMap<>(); for (Map.Entry<String, String> groupEntry : entry.getValue().entrySet()) { String groupId = groupEntry.getKey(); String segmentsString = groupEntry.getValue(); List<String> segments = Arrays.asList(segmentsString.split(SEGMENT_DELIMITER)); groupToSegmentsForLevel.put(groupId, new ArrayList<>(segments)); } groupToSegmentsMap.put(level, groupToSegmentsForLevel); } return new SegmentMergeLineage(tableNameWithType, segmentGroupLineageMap, groupToSegmentsMap); }
public int getNumPartitionsFromIdealState(IdealState idealState) { Set<Integer> partitions = new HashSet<>(); Map<String, Map<String, String>> mapFields = idealState.getRecord().getMapFields(); for (Map.Entry<String, Map<String, String>> entry : mapFields.entrySet()) { String segmentName = entry.getKey(); if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); partitions.add(llcSegmentName.getPartitionId()); } } return partitions.size(); }
/** * Gets the next ideal state based on the target (rebalanced) state. If no downtime is desired, the next state * is set such that there is always atleast one common replica for each segment between current and next state. */ private IdealState getNextState(IdealState currentState, IdealState targetState, Configuration rebalanceUserConfig) { // make a copy of the ideal state so it can be updated IdealState idealStateCopy = HelixHelper.cloneIdealState(currentState); Map<String, Map<String, String>> currentMapFields = currentState.getRecord().getMapFields(); Map<String, Map<String, String>> targetMapFields = targetState.getRecord().getMapFields(); for (String segmentId : targetMapFields.keySet()) { updateSegmentIfNeeded(segmentId, currentMapFields.get(segmentId), targetMapFields.get(segmentId), idealStateCopy, rebalanceUserConfig); } return idealStateCopy; }
/** * return true if IdealState = ExternalView * @return */ public int isStable(String tableName) { IdealState idealState = helixAdmin.getResourceIdealState(clusterName, tableName); ExternalView externalView = helixAdmin.getResourceExternalView(clusterName, tableName); Map<String, Map<String, String>> mapFieldsIS = idealState.getRecord().getMapFields(); Map<String, Map<String, String>> mapFieldsEV = externalView.getRecord().getMapFields(); int numDiff = 0; for (String segment : mapFieldsIS.keySet()) { Map<String, String> mapIS = mapFieldsIS.get(segment); Map<String, String> mapEV = mapFieldsEV.get(segment); for (String server : mapIS.keySet()) { String state = mapIS.get(server); if (mapEV == null || mapEV.get(server) == null || !mapEV.get(server).equals(state)) { LOGGER.info( "Mismatch: segment " + segment + " server:" + server + " expected state:" + state + " actual state:" + ( (mapEV == null || mapEV.get(server) == null) ? "null" : mapEV.get(server))); numDiff = numDiff + 1; } } } return numDiff; }
/** * Check if IdealState = ExternalView. If its not equal, return the number of differing segments. */ public int isStable(String tableName) { IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName); ExternalView externalView = _helixAdmin.getResourceExternalView(_helixClusterName, tableName); Map<String, Map<String, String>> mapFieldsIS = idealState.getRecord().getMapFields(); Map<String, Map<String, String>> mapFieldsEV = externalView.getRecord().getMapFields(); int numDiff = 0; for (String segment : mapFieldsIS.keySet()) { Map<String, String> mapIS = mapFieldsIS.get(segment); Map<String, String> mapEV = mapFieldsEV.get(segment); for (String server : mapIS.keySet()) { String state = mapIS.get(server); if (mapEV == null || mapEV.get(server) == null || !mapEV.get(server).equals(state)) { LOGGER.debug("Mismatch: segment" + segment + " server:" + server + " state:" + state); numDiff = numDiff + 1; } } } return numDiff; }
/** * return true if IdealState = ExternalView * @return */ public int isStable(String tableName) { IdealState idealState = helixAdmin.getResourceIdealState(clusterName, tableName); ExternalView externalView = helixAdmin.getResourceExternalView(clusterName, tableName); Map<String, Map<String, String>> mapFieldsIS = idealState.getRecord().getMapFields(); Map<String, Map<String, String>> mapFieldsEV = externalView.getRecord().getMapFields(); int numDiff = 0; for (String segment : mapFieldsIS.keySet()) { Map<String, String> mapIS = mapFieldsIS.get(segment); Map<String, String> mapEV = mapFieldsEV.get(segment); for (String server : mapIS.keySet()) { String state = mapIS.get(server); if (mapEV == null || mapEV.get(server) == null || !mapEV.get(server).equals(state)) { LOGGER.info("Mismatch: segment" + segment + " server:" + server + " state:" + state); numDiff = numDiff + 1; } } } return numDiff; }
@Nullable public Map<String, Map<String, String>> getIdealState(@Nonnull String tableNameOptType, @Nullable CommonConstants.Helix.TableType tableType) { String tableNameWithType = getTableNameWithType(tableNameOptType, tableType); IdealState resourceIdealState = _pinotHelixResourceManager.getHelixAdmin() .getResourceIdealState(_pinotHelixResourceManager.getHelixClusterName(), tableNameWithType); return resourceIdealState == null ? null : resourceIdealState.getRecord().getMapFields(); }
/** * Gets stream partition assignment of a table by reading the segment assignment in ideal state */ public PartitionAssignment getStreamPartitionAssignmentFromIdealState(TableConfig tableConfig, IdealState idealState) { String tableNameWithType = tableConfig.getTableName(); // get latest segment in each partition Map<String, LLCSegmentName> partitionIdToLatestSegment = getPartitionToLatestSegments(idealState); // extract partition assignment from the latest segments PartitionAssignment partitionAssignment = new PartitionAssignment(tableNameWithType); Map<String, Map<String, String>> mapFields = idealState.getRecord().getMapFields(); for (Map.Entry<String, LLCSegmentName> entry : partitionIdToLatestSegment.entrySet()) { String segmentName = entry.getValue().getSegmentName(); Map<String, String> instanceStateMap = mapFields.get(segmentName); partitionAssignment.addPartition(entry.getKey(), Lists.newArrayList(instanceStateMap.keySet())); } return partitionAssignment; }
@Nullable public Map<String, Map<String, String>> getExternalView(@Nonnull String tableNameOptType, CommonConstants.Helix.TableType tableType) { String tableNameWithType = getTableNameWithType(tableNameOptType, tableType); ExternalView resourceEV = _pinotHelixResourceManager.getHelixAdmin() .getResourceExternalView(_pinotHelixResourceManager.getHelixClusterName(), tableNameWithType); return resourceEV == null ? null : resourceEV.getRecord().getMapFields(); }
public IdealStateBuilderUtil clear() { _idealState.getRecord().getMapFields().clear(); return this; } }
/** * Generates a map of partition id to latest llc segment * @param idealState * @return */ @VisibleForTesting public Map<String, LLCSegmentName> getPartitionToLatestSegments(IdealState idealState) { Map<String, LLCSegmentName> partitionIdToLatestSegment = new HashMap<>(); // read all segments Map<String, Map<String, String>> mapFields = idealState.getRecord().getMapFields(); // get latest segment in each partition for (Map.Entry<String, Map<String, String>> entry : mapFields.entrySet()) { String segmentName = entry.getKey(); if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); String partitionId = String.valueOf(llcSegmentName.getPartitionId()); LLCSegmentName latestSegment = partitionIdToLatestSegment.get(partitionId); if (latestSegment == null || llcSegmentName.getSequenceNumber() > latestSegment.getSequenceNumber()) { partitionIdToLatestSegment.put(partitionId, llcSegmentName); } } } return partitionIdToLatestSegment; }
public IdealStateBuilderUtil removeSegment(String segmentName) { _idealState.getRecord().getMapFields().remove(segmentName); return this; }
/** * Verifies that all entries in old ideal state are unchanged in the new ideal state * There could be new entries in the ideal state due to num partitions increase * @param oldMapFields * @param idealState */ private void verifyNoChangeToOldEntries(Map<String, Map<String, String>> oldMapFields, IdealState idealState) { Map<String, Map<String, String>> newMapFields = idealState.getRecord().getMapFields(); for (Map.Entry<String, Map<String, String>> oldMapFieldsEntry : oldMapFields.entrySet()) { String oldSegment = oldMapFieldsEntry.getKey(); Map<String, String> oldInstanceStateMap = oldMapFieldsEntry.getValue(); Assert.assertTrue(newMapFields.containsKey(oldSegment)); Assert.assertTrue(oldInstanceStateMap.equals(newMapFields.get(oldSegment))); } }
private boolean validateNumSegments(int numSegments) { String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME); IdealState idealState = _helixAdmin.getResourceIdealState(getHelixClusterName(), tableNameWithType); return idealState.getRecord().getMapFields().keySet().size() == numSegments; }
public List<String> getInstances(int partition, int seqNum) { List<String> instances = new ArrayList<>(); for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { Map<String, String> instanceStateMap = _idealState.getInstanceStateMap(segmentName); instances = Lists.newArrayList(instanceStateMap.keySet()); break; } } } return instances; }
public IdealStateBuilderUtil setSegmentState(int partition, int seqNum, String state) { for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { Map<String, String> instanceStateMap = _idealState.getInstanceStateMap(segmentName); for (Map.Entry<String, String> entry : instanceStateMap.entrySet()) { instanceStateMap.put(entry.getKey(), state); } break; } } } return this; }
public IdealStateBuilderUtil moveToServers(int partition, int seqNum, List<String> instances) { for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { Map<String, String> instanceStateMap = _idealState.getInstanceStateMap(segmentName); Map<String, String> newInstanceStateMap = new HashMap<>(instanceStateMap.size()); int serverId = 0; for (Map.Entry<String, String> entry : instanceStateMap.entrySet()) { newInstanceStateMap.put(instances.get(serverId++), entry.getValue()); } _idealState.setInstanceStateMap(llcSegmentName.getSegmentName(), newInstanceStateMap); break; } } } return this; }
public String getSegment(int partition, int seqNum) { for (String segmentName : _idealState.getRecord().getMapFields().keySet()) { if (LLCSegmentName.isLowLevelConsumerSegmentName(segmentName)) { LLCSegmentName llcSegmentName = new LLCSegmentName(segmentName); if (llcSegmentName.getPartitionId() == partition && llcSegmentName.getSequenceNumber() == seqNum) { return segmentName; } } } return null; }
private boolean validateTableLevelReplicaGroupRebalance() { TableConfig tableConfig = _helixResourceManager.getTableConfig(TABLE_NAME, CommonConstants.Helix.TableType.OFFLINE); String tableNameWithType = TableNameBuilder.OFFLINE.tableNameWithType(TABLE_NAME); ReplicaGroupPartitionAssignmentGenerator partitionAssignmentGenerator = new ReplicaGroupPartitionAssignmentGenerator(_propertyStore); ReplicaGroupPartitionAssignment replicaGroupMapping = partitionAssignmentGenerator.getReplicaGroupPartitionAssignment(tableNameWithType); IdealState idealState = _helixAdmin.getResourceIdealState(getHelixClusterName(), tableNameWithType); Map<String, Map<String, String>> segmentAssignment = idealState.getRecord().getMapFields(); Map<Integer, Set<String>> segmentsPerPartition = new HashMap<>(); segmentsPerPartition.put(0, segmentAssignment.keySet()); return ReplicaGroupTestUtils .validateReplicaGroupSegmentAssignment(tableConfig, replicaGroupMapping, segmentAssignment, segmentsPerPartition); }