instanceAssignments = strategy.assign(Lists.newArrayList(newSegmentId), partitionAssignment); } catch (InvalidConfigException e) { _controllerMetrics.addMeteredTableValue(idealState.getResourceName(), ControllerMeter.CONTROLLER_REALTIME_TABLE_SEGMENT_ASSIGNMENT_ERROR, 1L); throw new IllegalStateException("Caught exception when updating ideal state on segment completion", e);
idealState.getResourceName(), segmentName, instanceStateMap.size(), targetNumReplicas);
LOGGER.info("Rebalancing consuming segments for table {}", idealState.getResourceName());
/** * Get the number of replicas for each partition of this resource * * @return number of replicas */ public int getReplicaCount(int eligibleInstancesCount) { String replicaStr = getReplicas(); int replica = 0; try { replica = Integer.parseInt(replicaStr); } catch (NumberFormatException ex) { if (replicaStr .equalsIgnoreCase(ResourceConfig.ResourceConfigConstants.ANY_LIVEINSTANCE.name())) { replica = eligibleInstancesCount; } else { logger.error("Can not determine the replica count for resource " + getResourceName() + ", set to 0."); } } return replica; }
/** * Get the number of replicas for each partition of this resource * * @return number of replicas */ public int getReplicaCount(int eligibleInstancesCount) { String replicaStr = getReplicas(); int replica = 0; try { replica = Integer.parseInt(replicaStr); } catch (NumberFormatException ex) { if (replicaStr .equalsIgnoreCase(ResourceConfig.ResourceConfigConstants.ANY_LIVEINSTANCE.name())) { replica = eligibleInstancesCount; } else { logger.error("Can not determine the replica count for resource " + getResourceName() + ", set to 0."); } } return replica; }
public void setResourceStatus(ExternalView externalView, IdealState idealState, StateModelDefinition stateModelDef) { try { ResourceMonitor resourceMonitor = getOrCreateResourceMonitor(externalView.getId()); if (resourceMonitor != null) { resourceMonitor.updateResource(externalView, idealState, stateModelDef); } } catch (Exception e) { LOG.error("Fail to set resource status, resource: " + idealState.getResourceName(), e); } }
private ZNRecord emptyMapping(IdealState idealState) { ZNRecord emptyMapping = new ZNRecord(idealState.getResourceName()); for (String partition : idealState.getPartitionSet()) { emptyMapping.setListField(partition, new ArrayList<String>()); } return emptyMapping; }
private ZNRecord emptyMapping(IdealState idealState) { ZNRecord emptyMapping = new ZNRecord(idealState.getResourceName()); for (String partition : idealState.getPartitionSet()) { emptyMapping.setListField(partition, new ArrayList<String>()); } return emptyMapping; }
public void setResourceStatus(ExternalView externalView, IdealState idealState, StateModelDefinition stateModelDef, int messageCount) { try { ResourceMonitor resourceMonitor = getOrCreateResourceMonitor(externalView.getId()); if (resourceMonitor != null) { resourceMonitor.updateResourceState(externalView, idealState, stateModelDef); resourceMonitor.updatePendingStateTransitionMessages(messageCount); } } catch (Exception e) { LOG.error("Fail to set resource status, resource: " + idealState.getResourceName(), e); } }
/** * Get all of the partitions * @return a set of partition names */ public Set<String> getPartitionSet() { if (getRebalanceMode() == RebalanceMode.SEMI_AUTO || getRebalanceMode() == RebalanceMode.FULL_AUTO || getRebalanceMode() == RebalanceMode.USER_DEFINED || getRebalanceMode() == RebalanceMode.TASK) { return _record.getListFields().keySet(); } else if (getRebalanceMode() == RebalanceMode.CUSTOMIZED) { return _record.getMapFields().keySet(); } else { logger.error("Invalid ideal state mode:" + getResourceName()); return Collections.emptySet(); } }
/** * Get all of the partitions * @return a set of partition names */ public Set<String> getPartitionSet() { if (getRebalanceMode() == RebalanceMode.SEMI_AUTO || getRebalanceMode() == RebalanceMode.FULL_AUTO || getRebalanceMode() == RebalanceMode.USER_DEFINED || getRebalanceMode() == RebalanceMode.TASK) { return _record.getListFields().keySet(); } else if (getRebalanceMode() == RebalanceMode.CUSTOMIZED) { return _record.getMapFields().keySet(); } else { logger.error("Invalid ideal state mode:" + getResourceName()); return Collections.emptySet(); } }
private void validateMapping(IdealState idealState, RoutingTableSnapshot routingTableSnapshot) { String db = idealState.getResourceName(); Set<String> partitions = idealState.getPartitionSet(); for (String partition : partitions) { List<InstanceConfig> masterInsEv = routingTableSnapshot.getInstancesForResource(db, partition, "MASTER"); Assert.assertEquals(masterInsEv.size(), 1); List<InstanceConfig> slaveInsEv = routingTableSnapshot.getInstancesForResource(db, partition, "SLAVE"); Assert.assertEquals(slaveInsEv.size(), 2); } } }
protected Map<String, String> computeBestPossibleStateForPartition(Set<String> liveInstances, StateModelDefinition stateModelDef, List<String> preferenceList, CurrentStateOutput currentStateOutput, Set<String> disabledInstancesForPartition, IdealState idealState, ClusterConfig clusterConfig, Partition partition) { Map<String, String> currentStateMap = currentStateOutput.getCurrentStateMap(idealState.getResourceName(), partition); if (currentStateMap == null) { currentStateMap = Collections.emptyMap(); } // (1) If the partition is removed from IS or the IS is deleted. // Transit to DROPPED no matter the instance is disabled or not. if (preferenceList == null) { return computeBestPossibleMapForDroppedResource(currentStateMap); } // (2) If resource disabled altogether, transit to initial-state (e.g. OFFLINE) if it's not in ERROR. if (!idealState.isEnabled()) { return computeBestPossibleMapForDisabledResource(currentStateMap, stateModelDef); } return computeBestPossibleMap(preferenceList, stateModelDef, currentStateMap, liveInstances, disabledInstancesForPartition); }
protected Map<String, String> computeBestPossibleStateForPartition(Set<String> liveInstances, StateModelDefinition stateModelDef, List<String> preferenceList, CurrentStateOutput currentStateOutput, Set<String> disabledInstancesForPartition, IdealState idealState, ClusterConfig clusterConfig, Partition partition) { Map<String, String> currentStateMap = currentStateOutput.getCurrentStateMap(idealState.getResourceName(), partition); if (currentStateMap == null) { currentStateMap = Collections.emptyMap(); } // (1) If the partition is removed from IS or the IS is deleted. // Transit to DROPPED no matter the instance is disabled or not. if (preferenceList == null) { return computeBestPossibleMapForDroppedResource(currentStateMap); } // (2) If resource disabled altogether, transit to initial-state (e.g. OFFLINE) if it's not in ERROR. if (!idealState.isEnabled()) { return computeBestPossibleMapForDisabledResource(currentStateMap, stateModelDef); } return computeBestPossibleMap(preferenceList, stateModelDef, currentStateMap, liveInstances, disabledInstancesForPartition); }
private void validate(IdealState idealState, RoutingTableProvider routingTableEV, RoutingTableProvider routingTableCurrentStates) { String db = idealState.getResourceName(); Set<String> partitions = idealState.getPartitionSet(); for (String partition : partitions) { List<InstanceConfig> masterInsEv = routingTableEV.getInstancesForResource(db, partition, "MASTER"); List<InstanceConfig> masterInsCs = routingTableCurrentStates.getInstancesForResource(db, partition, "MASTER"); Assert.assertEquals(masterInsEv.size(), 1); Assert.assertEquals(masterInsCs.size(), 1); Assert.assertEquals(masterInsCs, masterInsEv); List<InstanceConfig> slaveInsEv = routingTableEV.getInstancesForResource(db, partition, "SLAVE"); List<InstanceConfig> slaveInsCs = routingTableCurrentStates.getInstancesForResource(db, partition, "SLAVE"); Assert.assertEquals(slaveInsEv.size(), 2); Assert.assertEquals(slaveInsCs.size(), 2); Assert.assertEquals(new HashSet(slaveInsCs), new HashSet(slaveInsEv)); } } }
@Override public void onIdealStateChange(List<IdealState> idealStates, NotificationContext changeContext) { if (!_startListen) { return; } for (IdealState is : idealStates) { ExternalView view = _gSetupTool.getClusterManagementTool() .getResourceExternalView(CLUSTER_NAME, is.getResourceName()); validateNoZeroReplica(is, view); } }
protected void setSingleIdealState(IdealState idealState) { accessor .setProperty(accessor.keyBuilder().idealStates(idealState.getResourceName()), idealState); }
/** * Validate instances for each partition is on different zone and with necessary tagged * instances. */ private void validateNoZeroReplica(IdealState is, ExternalView ev) { int replica = is.getReplicaCount(NUM_NODE); StateModelDefinition stateModelDef = BuiltInStateModelDefinitions.valueOf(is.getStateModelDefRef()).getStateModelDefinition(); for (String partition : is.getPartitionSet()) { Map<String, String> evStateMap = ev.getRecord().getMapField(partition); Map<String, String> isStateMap = is.getInstanceStateMap(partition); validateMap(is.getResourceName(), partition, replica, evStateMap, stateModelDef); validateMap(is.getResourceName(), partition, replica, isStateMap, stateModelDef); } }
@Override public void onExternalViewChange(List<ExternalView> externalViewList, NotificationContext changeContext) { if (!trackEnabled) { return; } for (ExternalView ev : externalViewList) { IdealState is = _resourceMap.get(ev.getResourceName()); if (is == null) { continue; } int replica = is.getReplicaCount(NUM_NODE); for (String p : is.getPartitionSet()) { Map<String, String> stateMap = ev.getStateMap(p); verifyPartitionCount(is.getResourceName(), p, stateMap, replica, "EV", is.getMinActiveReplicas()); } } }
@Override public void onIdealStateChange(List<IdealState> idealStates, NotificationContext changeContext) throws InterruptedException { if (!trackEnabled) { return; } for (IdealState is : idealStates) { int replica = is.getReplicaCount(NUM_NODE); for (String p : is.getPartitionSet()) { Map<String, String> stateMap = is.getRecord().getMapField(p); verifyPartitionCount(is.getResourceName(), p, stateMap, replica, "IS", is.getMinActiveReplicas()); } } }