private void persistHistory(ParticipantHistory history) { PropertyKey propertyKey = _keyBuilder.participantHistory(_instanceName); if (!_dataAccessor.setProperty(propertyKey, history)) { LOG.error("Failed to persist participant history to zk!"); } }
private void persistHistory(ParticipantHistory history) { PropertyKey propertyKey = _keyBuilder.participantHistory(_instanceName); if (!_dataAccessor.setProperty(propertyKey, history)) { LOG.error("Failed to persist participant history to zk!"); } }
@Override public void addClusterToGrandCluster(String clusterName, String grandCluster) { logger.info("Add cluster {} to grand cluster {}.", clusterName, grandCluster); if (!ZKUtil.isClusterSetup(grandCluster, _zkClient)) { throw new HelixException("Grand cluster " + grandCluster + " is not setup yet"); } if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { throw new HelixException("Cluster " + clusterName + " is not setup yet"); } IdealState idealState = new IdealState(clusterName); idealState.setNumPartitions(1); idealState.setStateModelDefRef("LeaderStandby"); List<String> controllers = getInstancesInCluster(grandCluster); if (controllers.size() == 0) { throw new HelixException("Grand cluster " + grandCluster + " has no instances"); } idealState.setReplicas(Integer.toString(controllers.size())); Collections.shuffle(controllers); idealState.getRecord().setListField(clusterName, controllers); idealState.setPartitionState(clusterName, controllers.get(0), "LEADER"); for (int i = 1; i < controllers.size(); i++) { idealState.setPartitionState(clusterName, controllers.get(i), "STANDBY"); } ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(grandCluster, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); Builder keyBuilder = accessor.keyBuilder(); accessor.setProperty(keyBuilder.idealStates(idealState.getResourceName()), idealState); }
@Override public void addClusterToGrandCluster(String clusterName, String grandCluster) { logger.info("Add cluster {} to grand cluster {}.", clusterName, grandCluster); if (!ZKUtil.isClusterSetup(grandCluster, _zkClient)) { throw new HelixException("Grand cluster " + grandCluster + " is not setup yet"); } if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { throw new HelixException("Cluster " + clusterName + " is not setup yet"); } IdealState idealState = new IdealState(clusterName); idealState.setNumPartitions(1); idealState.setStateModelDefRef("LeaderStandby"); List<String> controllers = getInstancesInCluster(grandCluster); if (controllers.size() == 0) { throw new HelixException("Grand cluster " + grandCluster + " has no instances"); } idealState.setReplicas(Integer.toString(controllers.size())); Collections.shuffle(controllers); idealState.getRecord().setListField(clusterName, controllers); idealState.setPartitionState(clusterName, controllers.get(0), "LEADER"); for (int i = 1; i < controllers.size(); i++) { idealState.setPartitionState(clusterName, controllers.get(i), "STANDBY"); } ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(grandCluster, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); Builder keyBuilder = accessor.keyBuilder(); accessor.setProperty(keyBuilder.idealStates(idealState.getResourceName()), idealState); }
@Override public void removeInstanceTag(String clusterName, String instanceName, String tag) { logger.info("Remove instance tag {} for instance {} in cluster {}.", tag, instanceName, clusterName); if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { throw new HelixException("cluster " + clusterName + " is not setup yet"); } if (!ZKUtil.isInstanceSetup(_zkClient, clusterName, instanceName, InstanceType.PARTICIPANT)) { throw new HelixException( "cluster " + clusterName + " instance " + instanceName + " is not setup yet"); } ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); Builder keyBuilder = accessor.keyBuilder(); InstanceConfig config = accessor.getProperty(keyBuilder.instanceConfig(instanceName)); config.removeTag(tag); accessor.setProperty(keyBuilder.instanceConfig(instanceName), config); }
@Override public void removeInstanceTag(String clusterName, String instanceName, String tag) { logger.info("Remove instance tag {} for instance {} in cluster {}.", tag, instanceName, clusterName); if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) { throw new HelixException("cluster " + clusterName + " is not setup yet"); } if (!ZKUtil.isInstanceSetup(_zkClient, clusterName, instanceName, InstanceType.PARTICIPANT)) { throw new HelixException( "cluster " + clusterName + " instance " + instanceName + " is not setup yet"); } ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_zkClient)); Builder keyBuilder = accessor.keyBuilder(); InstanceConfig config = accessor.getProperty(keyBuilder.instanceConfig(instanceName)); config.removeTag(tag); accessor.setProperty(keyBuilder.instanceConfig(instanceName), config); }
protected List<IdealState> setupIdealState(String clusterName, int[] nodes, String[] resources, int partitions, int replicas) { ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient)); Builder keyBuilder = accessor.keyBuilder(); List<IdealState> idealStates = new ArrayList<IdealState>(); List<String> instances = new ArrayList<String>(); for (int i : nodes) { instances.add("localhost_" + i); } for (String resourceName : resources) { IdealState idealState = new IdealState(resourceName); for (int p = 0; p < partitions; p++) { List<String> value = new ArrayList<String>(); for (int r = 0; r < replicas; r++) { int n = nodes[(p + r) % nodes.length]; value.add("localhost_" + n); } idealState.getRecord().setListField(resourceName + "_" + p, value); } idealState.setReplicas(Integer.toString(replicas)); idealState.setStateModelDefRef("MasterSlave"); idealState.setRebalanceMode(IdealState.RebalanceMode.SEMI_AUTO); idealState.setNumPartitions(partitions); idealStates.add(idealState); // System.out.println(idealState); accessor.setProperty(keyBuilder.idealStates(resourceName), idealState); } return idealStates; }
@Test() public void testRenamePartitionAutoIS() throws Exception { String clusterName = "CLUSTER_" + getShortClassName() + "_auto"; System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis())); TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant start port "localhost", // participant name prefix "TestDB", // resource name prefix 1, // resources 10, // partitions per resource 5, // number of nodes 3, // replicas "MasterSlave", true); // do rebalance startAndVerify(clusterName); // rename partition name TestDB0_0 tp TestDB0_100 ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient)); Builder keyBuilder = accessor.keyBuilder(); IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0")); List<String> prioList = idealState.getRecord().getListFields().remove("TestDB0_0"); idealState.getRecord().getListFields().put("TestDB0_100", prioList); accessor.setProperty(keyBuilder.idealStates("TestDB0"), idealState); boolean result = ClusterStateVerifier.verifyByPolling(new ClusterStateVerifier.BestPossAndExtViewZkVerifier( ZK_ADDR, clusterName)); Assert.assertTrue(result); stop(clusterName); System.out.println("END " + clusterName + " at " + new Date(System.currentTimeMillis())); }
protected void setupStateModel(String clusterName) { ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient)); Builder keyBuilder = accessor.keyBuilder(); StateModelDefinition masterSlave = new StateModelDefinition(StateModelConfigGenerator.generateConfigForMasterSlave()); accessor.setProperty(keyBuilder.stateModelDef(masterSlave.getId()), masterSlave); StateModelDefinition leaderStandby = new StateModelDefinition(StateModelConfigGenerator.generateConfigForLeaderStandby()); accessor.setProperty(keyBuilder.stateModelDef(leaderStandby.getId()), leaderStandby); StateModelDefinition onlineOffline = new StateModelDefinition(StateModelConfigGenerator.generateConfigForOnlineOffline()); accessor.setProperty(keyBuilder.stateModelDef(onlineOffline.getId()), onlineOffline); }
Builder keyBuilder = accessor.keyBuilder(); accessor.setProperty(keyBuilder.idealStates("TestDB0"), idealState); accessor.setProperty(keyBuilder.idealStates("TestDB0"), idealState);
private void setupCluster(String clusterName, ZKHelixDataAccessor accessor) throws Exception { TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant start port "localhost", // participant name prefix resourceNamePrefix, // resource name prefix 1, // resources 15, // partitions per resource participantCount, // number of nodes 3, // replicas "MasterSlave", IdealState.RebalanceMode.FULL_AUTO, true); // do rebalance PropertyKey.Builder keyBuilder = accessor.keyBuilder(); ClusterConfig clusterConfig = accessor.getProperty(accessor.keyBuilder().clusterConfig()); clusterConfig.setResourcePriorityField("Name"); List<StateTransitionThrottleConfig> throttleConfigs = new ArrayList<>(); throttleConfigs.add( new StateTransitionThrottleConfig(StateTransitionThrottleConfig.RebalanceType.LOAD_BALANCE, StateTransitionThrottleConfig.ThrottleScope.CLUSTER, 100)); throttleConfigs.add(new StateTransitionThrottleConfig( StateTransitionThrottleConfig.RebalanceType.RECOVERY_BALANCE, StateTransitionThrottleConfig.ThrottleScope.CLUSTER, 100)); clusterConfig.setStateTransitionThrottleConfigs(throttleConfigs); accessor.setProperty(keyBuilder.clusterConfig(), clusterConfig); }
private void setupCluster(String clusterName, List<String> instanceNames, String dbName, int replica, int partitions, int bucketSize) { _gSetupTool.addCluster(clusterName, true); _gSetupTool.addInstancesToCluster(clusterName, instanceNames.toArray(new String[instanceNames.size()])); // add a bucketized resource ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, _baseAccessor); PropertyKey.Builder keyBuilder = accessor.keyBuilder(); ZNRecord idealStateRec = DefaultIdealStateCalculator.calculateIdealState(instanceNames, partitions, replica - 1, dbName, "MASTER", "SLAVE"); IdealState idealState = new IdealState(idealStateRec); idealState.setBucketSize(bucketSize); idealState.setStateModelDefRef("MasterSlave"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED); idealState.setReplicas(Integer.toString(replica)); accessor.setProperty(keyBuilder.idealStates(dbName), idealState); }
IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0")); idealState.setRebalanceMode(RebalanceMode.CUSTOMIZED); accessor.setProperty(keyBuilder.idealStates("TestDB0"), idealState);
public void createExternalView(List<String> instanceNames, int partitions, int replicas, String dbName, long randomSeed) { ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor(client)); Builder keyBuilder = accessor.keyBuilder(); ExternalView externalView = new ExternalView(computeRoutingTable(instanceNames, partitions, replicas, dbName, randomSeed)); accessor.setProperty(keyBuilder.externalView(dbName), externalView); }
private void setCurrentState(String clusterName, String instance, String resourceGroupName, String resourceKey, String sessionId, String state, boolean updateTimestamp) { ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient)); Builder keyBuilder = accessor.keyBuilder(); CurrentState curState = new CurrentState(resourceGroupName); curState.setState(resourceKey, state); curState.setSessionId(sessionId); curState.setStateModelDefRef("MasterSlave"); if (updateTimestamp) { curState.setEndTime(resourceKey, System.currentTimeMillis()); } accessor.setProperty(keyBuilder.currentState(instance, sessionId, resourceGroupName), curState); }
protected void setupLiveInstances(String clusterName, int[] liveInstances) { ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient)); Builder keyBuilder = accessor.keyBuilder(); for (int i = 0; i < liveInstances.length; i++) { String instance = "localhost_" + liveInstances[i]; LiveInstance liveInstance = new LiveInstance(instance); liveInstance.setSessionId("session_" + liveInstances[i]); liveInstance.setHelixVersion("0.0.0"); accessor.setProperty(keyBuilder.liveInstance(instance), liveInstance); } }
liveInstance.setSessionId("session_0"); liveInstance.setHelixVersion("version_0"); accessor.setProperty(keyBuilder.liveInstance("localhost_12918"), liveInstance);
accessor.setProperty(keyBuilder.liveInstance(instance), liveInstance);
idealState.getRecord() .setListField("TestDB0_0", Arrays.asList(HelixConstants.StateModelToken.ANY_LIVEINSTANCE.toString())); accessor.setProperty(key, idealState);
IdealState idealState = accessor.getProperty(keyBuilder.idealStates("TestDB0")); idealState.setBatchMessageMode(true); accessor.setProperty(keyBuilder.idealStates("TestDB0"), idealState);