@Nullable @Override public IdealState apply(@Nullable IdealState idealState) { idealState.setReplicas(configReplication); return idealState; } }, RetryPolicies.exponentialBackoffRetryPolicy(5, 1000L, 1.2f));
private IdealState updateIdealState(IdealState idealState, int newNumReplicas) { idealState.setReplicas(Integer.toString(newNumReplicas)); Set<String> segmentIds = idealState.getPartitionSet(); for (String segmentId : segmentIds) { Map<String, String> instanceStateMap = idealState.getInstanceStateMap(segmentId); if (instanceStateMap.size() > newNumReplicas) { Set<String> keys = instanceStateMap.keySet(); while (instanceStateMap.size() > newNumReplicas) { instanceStateMap.remove(keys.iterator().next()); } } else if (instanceStateMap.size() < newNumReplicas) { throw new RuntimeException( "Segment " + segmentId + " has " + instanceStateMap.size() + " replicas but want changed to " + newNumReplicas); } } return idealState; }
public IdealStateBuilderUtil setNumReplicas(int numReplicas) { _idealState.setReplicas(String.valueOf(numReplicas)); return this; }
/** * Rebalances ideal state object without updating it * @param idealState * @param tableConfig * @param targetNumReplicas * @param rebalanceUserConfig * @param newPartitionAssignment */ private IdealState rebalanceIdealState(IdealState idealState, TableConfig tableConfig, int targetNumReplicas, Configuration rebalanceUserConfig, PartitionAssignment newPartitionAssignment) { // if realtime and includeConsuming, then rebalance consuming segments if (tableConfig.getTableType().equals(CommonConstants.Helix.TableType.REALTIME)) { boolean includeConsuming = rebalanceUserConfig.getBoolean(RebalanceUserConfigConstants.INCLUDE_CONSUMING, DEFAULT_INCLUDE_CONSUMING); if (includeConsuming) { rebalanceConsumingSegments(idealState, newPartitionAssignment); } } // always rebalance serving segments rebalanceServingSegments(idealState, tableConfig, targetNumReplicas); idealState.setReplicas(Integer.toString(targetNumReplicas)); return idealState; }
idealState.setInstanceStateMap(entry.getKey(), entry.getValue()); idealState.setReplicas(Integer.toString(numReplicaGroups));
idealState.setReplicas("1"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
allTableNames.add(tableName); IdealState idealState = new IdealState(tableName); idealState.setReplicas(nReplicasStr); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
idealState.setPartitionState("myTable_OFFLINE", "pinot2", "OFFLINE"); idealState.setPartitionState("myTable_OFFLINE", "pinot3", "OFFLINE"); idealState.setReplicas("1"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
idealState.setPartitionState("myTable_0", "pinot2", "OFFLINE"); idealState.setPartitionState("myTable_0", "pinot3", "OFFLINE"); idealState.setReplicas("0"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
idealState.setPartitionState("myTable_1", "pinot3", "ONLINE"); idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE"); idealState.setReplicas("2"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
/** * Regression test for large ideal state updates failing silently */ @Test public void testWriteLargeIdealState() { final int numSegments = 20000; IdealState idealState = new IdealState(RESOURCE_NAME); idealState.setStateModelDefRef("OnlineOffline"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED); idealState.setReplicas("0"); String helixClusterName = getHelixClusterName(); _helixAdmin.addResource(helixClusterName, RESOURCE_NAME, idealState); HelixHelper.updateIdealState(_helixManager, RESOURCE_NAME, new Function<IdealState, IdealState>() { @Override public IdealState apply(@Nullable IdealState idealState) { Assert.assertNotNull(idealState); for (int i = 0; i < numSegments; i++) { idealState.setPartitionState("segment_" + i, INSTANCE_NAME, "ONLINE"); } return idealState; } }, RetryPolicies.noDelayRetryPolicy(1)); IdealState resourceIdealState = _helixAdmin.getResourceIdealState(helixClusterName, RESOURCE_NAME); for (int i = 0; i < numSegments; i++) { Assert.assertEquals(resourceIdealState.getInstanceStateMap("segment_" + i).get(INSTANCE_NAME), "ONLINE"); } }
idealState.setPartitionState("myTable_1", "pinot3", "ONLINE"); idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE"); idealState.setReplicas("2"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
idealState.setPartitionState("myTable_1", "pinot1", "ONLINE"); idealState.setPartitionState("myTable_1", "pinot2", "ONLINE"); idealState.setReplicas("2"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
idealState.setPartitionState("myTable_2", "pinot3", "OFFLINE"); idealState.setPartitionState("myTable_3", "pinot3", "ONLINE"); idealState.setReplicas("2"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
idealState.setPartitionState(seg3.getSegmentName(), "pinot2", "CONSUMING"); idealState.setPartitionState(seg3.getSegmentName(), "pinot3", "OFFLINE"); idealState.setReplicas("3"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
@Override public IdealState computeNewIdealState(String resourceName, IdealState currentIdealState, CurrentStateOutput currentStateOutput, ClusterDataCache clusterData) { testRebalancerInvoked = true; List<String> liveNodes = Lists.newArrayList(clusterData.getLiveInstances().keySet()); int i = 0; for (String partition : currentIdealState.getPartitionSet()) { int index = i++ % liveNodes.size(); String instance = liveNodes.get(index); currentIdealState.getPreferenceList(partition).clear(); currentIdealState.getPreferenceList(partition).add(instance); currentIdealState.getInstanceStateMap(partition).clear(); currentIdealState.getInstanceStateMap(partition).put(instance, "MASTER"); } currentIdealState.setReplicas("1"); return currentIdealState; } }
@Test(dataProvider = "IdealStateInput") public void testIdealStateAssignment(String clusterName, List<String> instances, List<String> partitions, String numReplicas, String stateModeDef, String strategyName, Map<String, Map<String, String>> expectedMapping) throws IllegalAccessException, InstantiationException, ClassNotFoundException { ClusterConfig clusterConfig = new ClusterConfig(clusterName); List<InstanceConfig> instanceConfigs = new ArrayList<>(); for (String instance : instances) { instanceConfigs.add(new InstanceConfig(instance)); } IdealState idealState = new IdealState("TestResource"); idealState.setStateModelDefRef(stateModeDef); idealState.setNumPartitions(partitions.size()); idealState.setReplicas(numReplicas); Map<String, Map<String, String>> idealStateMapping = HelixUtil .getIdealAssignmentForFullAuto(clusterConfig, instanceConfigs, instances, idealState, partitions, strategyName); Assert.assertEquals(idealStateMapping, expectedMapping); }
@Test public void testReplicas() { IdealState idealState = new IdealState("test-db"); idealState.setRebalanceMode(RebalanceMode.SEMI_AUTO); idealState.setNumPartitions(4); idealState.setStateModelDefRef("MasterSlave"); idealState.setReplicas("" + 2); List<String> preferenceList = new ArrayList<String>(); preferenceList.add("node_0"); idealState.getRecord().setListField("test-db_0", preferenceList); Assert.assertFalse(idealState.isValid(), "should fail since replicas not equals to preference-list size"); preferenceList.add("node_1"); idealState.getRecord().setListField("test-db_0", preferenceList); Assert.assertTrue(idealState.isValid(), "should pass since replicas equals to preference-list size"); }
private void createIS(HelixDataAccessor accessor, String resourceId, String stateModelDefRef, RebalanceMode rebalanceMode) { IdealState idealState = new IdealState(resourceId); idealState.setRebalanceMode(rebalanceMode); idealState.setStateModelDefRef(stateModelDefRef); idealState.setNumPartitions(1); idealState.setReplicas("1"); idealState.getRecord().setListField(resourceId + "_0", ImmutableList.of(PARTICIPANT)); idealState.getRecord().setMapField(resourceId + "_0", ImmutableMap.of(PARTICIPANT, STATE)); accessor.setProperty(accessor.keyBuilder().idealStates(resourceId), idealState); }
private void setupCluster(String clusterName, List<String> instanceNames, String dbName, int replica, int partitions, int bucketSize) { _gSetupTool.addCluster(clusterName, true); _gSetupTool.addInstancesToCluster(clusterName, instanceNames.toArray(new String[instanceNames.size()])); // add a bucketized resource ZKHelixDataAccessor accessor = new ZKHelixDataAccessor(clusterName, _baseAccessor); PropertyKey.Builder keyBuilder = accessor.keyBuilder(); ZNRecord idealStateRec = DefaultIdealStateCalculator.calculateIdealState(instanceNames, partitions, replica - 1, dbName, "MASTER", "SLAVE"); IdealState idealState = new IdealState(idealStateRec); idealState.setBucketSize(bucketSize); idealState.setStateModelDefRef("MasterSlave"); idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED); idealState.setReplicas(Integer.toString(replica)); accessor.setProperty(keyBuilder.idealStates(dbName), idealState); }