/** * Toggle the state of the instance between OFFLINE and ONLINE. * * @param instanceName Name of the instance for which to toggle the state. * @param clusterName Name of the cluster to which the instance belongs. * @param admin HelixAdmin to access the cluster. * @param enable Set enable to true for ONLINE and FALSE for OFFLINE. */ public static void setInstanceState(String instanceName, String clusterName, HelixAdmin admin, boolean enable) { admin.enableInstance(clusterName, instanceName, enable); }
@Test(dataProvider = "tableNameProvider") public void testNoAssignmentToDisabledInstances(String tableName, SegmentVersion version) throws Exception { List<String> instances = _helixAdmin.getInstancesInClusterWithTag(getHelixClusterName(), serverTenant); List<String> disabledInstances = new ArrayList<>(); // disable 6 instances assertEquals(instances.size(), 9); for (int i = 0; i < 6; i++) { _helixAdmin.enableInstance(getHelixClusterName(), instances.get(i), false); } // Thread.sleep(100000); for (int i = 0; i < 6; i++) { generateAndUploadRandomSegment(tableName + "_" + i, 10); } Map<String, Integer> segmentsPerInstance = getSegmentsPerInstance(tableName); // size is 3 since we disabled 6 instances assertEquals(segmentsPerInstance.size(), 3); for (Map.Entry<String, Integer> instanceEntry : segmentsPerInstance.entrySet()) { assertEquals(instanceEntry.getValue().intValue(), 6); } // re-enable instances since these tests are usually "setup once" and run multiple tests type for (int i = 0; i < 6; i++) { _helixAdmin.enableInstance(getHelixClusterName(), instances.get(i), true); } }
_helixAdmin.enableInstance(_helixClusterName, instanceName, toggle); long deadline = System.currentTimeMillis() + 1000 * timeOutInSeconds; boolean toggleSucceed = false;
private void enableInstance(String instance, boolean enabled) { // Disable one node, no partition should be moved. long currentTime = System.currentTimeMillis(); _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, instance, enabled); InstanceConfig instanceConfig = _configAccessor.getInstanceConfig(CLUSTER_NAME, instance); Assert.assertEquals(instanceConfig.getInstanceEnabled(), enabled); Assert.assertTrue(instanceConfig.getInstanceEnabledTime() >= currentTime); Assert.assertTrue(instanceConfig.getInstanceEnabledTime() <= currentTime + 100); } }
@Test(dependsOnMethods = { "testDisableInstance" }) public void testRoutingTableListener() throws InterruptedException { RoutingTableChangeListener routingTableChangeListener = new MockRoutingTableChangeListener(); Map<String, Set<String>> context = new HashMap<>(); context.put("MASTER", Sets.newSet(_instances.get(0))); context.put("SLAVE", Sets.newSet(_instances.get(1), _instances.get(2))); _routingTableProvider_default.addRoutingTableChangeListener(routingTableChangeListener, context); _routingTableProvider_default .addRoutingTableChangeListener(new MockRoutingTableChangeListener(), null); // reenable the master instance to cause change String prevMasterInstance = _instances.get(0); _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, prevMasterInstance, true); Assert.assertTrue(_clusterVerifier.verifyByPolling()); Assert.assertTrue(_listenerTestResult); }
@Test public void testResourceSubset() throws InterruptedException { // Ensure that this passes even when one resource is down _admin.enableInstance(_clusterName, "localhost_12918", false); Thread.sleep(1000); _admin.enableCluster(_clusterName, false); _admin.enableInstance(_clusterName, "localhost_12918", true); boolean result = ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR, _clusterName, null, Sets.newHashSet(RESOURCES[1]))); Assert.assertTrue(result); String[] args = { "--zkSvr", ZK_ADDR, "--cluster", _clusterName, "--resources", RESOURCES[1] }; result = ClusterStateVerifier.verifyState(args); Assert.assertTrue(result); // But the full cluster verification should fail boolean fullResult = new BestPossAndExtViewZkVerifier(ZK_ADDR, _clusterName).verify(); Assert.assertFalse(fullResult); _admin.enableCluster(_clusterName, true); } }
@Test(dependsOnMethods = { "testRoutingTable" }) public void testDisableInstance() throws InterruptedException { // disable the master instance String prevMasterInstance = _instances.get(0); _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, prevMasterInstance, false); Assert.assertTrue(_clusterVerifier.verifyByPolling()); validateRoutingTable(_routingTableProvider_default, Sets.newSet(_instances.get(1)), Sets.newSet(_instances.get(2))); validateRoutingTable(_routingTableProvider_ev, Sets.newSet(_instances.get(1)), Sets.newSet(_instances.get(2))); validateRoutingTable(_routingTableProvider_cs, Sets.newSet(_instances.get(1)), Sets.newSet(_instances.get(2))); }
@Test public void testP2PStateTransitionDisabled() { enableP2PInCluster(CLUSTER_NAME, _configAccessor, false); MockHelixTaskExecutor.resetStats(); // rolling upgrade the cluster for (String ins : _instances) { _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, ins, false); Assert.assertTrue(_clusterVerifier.verifyByPolling()); verifyP2PDisabled(); _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, ins, true); Assert.assertTrue(_clusterVerifier.verifyByPolling()); verifyP2PDisabled(); } Assert.assertEquals(MockHelixTaskExecutor.duplicatedMessagesInProgress, 0, "There are duplicated transition messages sent while participant is handling the state-transition!"); Assert.assertEquals(MockHelixTaskExecutor.duplicatedMessages, 0, "There are duplicated transition messages sent at same time!"); }
@Test (enabled = false) public void testOldEnableDisable() throws InterruptedException { _gSetupTool.getClusterManagementTool() .enableInstance(CLUSTER_NAME, _participants[0].getInstanceName(), false); Assert.assertTrue(_clusterVerifier.verify()); ExternalView externalView = _gSetupTool.getClusterManagementTool() .getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB); Assert.assertEquals(externalView.getRecord().getMapFields().size(), _numPartitions); for (Map<String, String> stateMap : externalView.getRecord().getMapFields().values()) { Assert.assertTrue(!stateMap.keySet().contains(_participants[0].getInstanceName())); } _gSetupTool.getClusterManagementTool() .enableInstance(CLUSTER_NAME, _participants[0].getInstanceName(), true); }
@Test (dependsOnMethods = {"testP2PStateTransitionDisabled"}) public void testP2PStateTransitionEnabled() { enableP2PInCluster(CLUSTER_NAME, _configAccessor, true); long startTime = System.currentTimeMillis(); MockHelixTaskExecutor.resetStats(); // rolling upgrade the cluster for (String ins : _instances) { _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, ins, false); Assert.assertTrue(_clusterVerifier.verifyByPolling()); verifyP2PEnabled(startTime); _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, ins, true); Assert.assertTrue(_clusterVerifier.verifyByPolling()); verifyP2PEnabled(startTime); } double ratio = ((double) p2pTrigged) / ((double) total); Assert.assertTrue(ratio > 0.6, String .format("Only %d out of %d percent transitions to Master were triggered by expected host!", p2pTrigged, total)); Assert.assertEquals(MockHelixTaskExecutor.duplicatedMessagesInProgress, 0, "There are duplicated transition messages sent while participant is handling the state-transition!"); Assert.assertEquals(MockHelixTaskExecutor.duplicatedMessages, 0, "There are duplicated transition messages sent at same time!"); }
@Test (enabled = false) public void testBatchEnableDisable() throws InterruptedException { _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, Arrays.asList(_participants[0].getInstanceName(), _participants[1].getInstanceName()), false); Assert.assertTrue(_clusterVerifier.verifyByPolling()); ExternalView externalView = _gSetupTool.getClusterManagementTool() .getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB); Assert.assertEquals(externalView.getRecord().getMapFields().size(), _numPartitions); for (Map<String, String> stateMap : externalView.getRecord().getMapFields().values()) { Assert.assertTrue(!stateMap.keySet().contains(_participants[0].getInstanceName())); Assert.assertTrue(!stateMap.keySet().contains(_participants[1].getInstanceName())); } _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, Arrays.asList(_participants[0].getInstanceName(), _participants[1].getInstanceName()), true); }
@Test (enabled = false) public void testOldDisableBatchEnable() throws InterruptedException { _gSetupTool.getClusterManagementTool() .enableInstance(CLUSTER_NAME, _participants[0].getInstanceName(), false); _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, Arrays.asList(_participants[0].getInstanceName(), _participants[1].getInstanceName()), true); Thread.sleep(2000); ExternalView externalView = _gSetupTool.getClusterManagementTool() .getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB); Assert.assertEquals(externalView.getRecord().getMapFields().size(), _numPartitions); int numOfFirstHost = 0; for (Map<String, String> stateMap : externalView.getRecord().getMapFields().values()) { if (stateMap.keySet().contains(_participants[0].getInstanceName())) { numOfFirstHost++; } } Assert.assertTrue(numOfFirstHost > 0); _gSetupTool.getClusterManagementTool() .enableInstance(CLUSTER_NAME, _participants[0].getInstanceName(), true); }
@Test (enabled = false) public void testBatchDisableOldEnable() throws InterruptedException { _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, Arrays.asList(_participants[0].getInstanceName(), _participants[1].getInstanceName()), false); _gSetupTool.getClusterManagementTool() .enableInstance(CLUSTER_NAME, _participants[0].getInstanceName(), true); Thread.sleep(2000); ExternalView externalView = _gSetupTool.getClusterManagementTool() .getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB); Assert.assertEquals(externalView.getRecord().getMapFields().size(), _numPartitions); int numOfFirstHost = 0; for (Map<String, String> stateMap : externalView.getRecord().getMapFields().values()) { if (stateMap.keySet().contains(_participants[0].getInstanceName())) { numOfFirstHost++; } Assert.assertTrue(!stateMap.keySet().contains(_participants[1].getInstanceName())); } Assert.assertTrue(numOfFirstHost > 0); _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, Arrays.asList(_participants[0].getInstanceName(), _participants[1].getInstanceName()), true); } }
@Test (dependsOnMethods = {"testP2PStateTransitionDisabled"}) public void testP2PStateTransitionEnabledInCluster() { enableP2PInCluster(CLUSTER_NAME, _configAccessor, true); enableP2PInResource(CLUSTER_NAME, _configAccessor, DB_NAME_1,false); enableP2PInResource(CLUSTER_NAME, _configAccessor, DB_NAME_2,false); // disable the master instance String prevMasterInstance = _instances.get(0); _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, prevMasterInstance, false); Assert.assertTrue(_clusterVerifier.verifyByPolling()); verifyP2PMessage(DB_NAME_1, _instances.get(1), MasterSlaveSMD.States.MASTER.name(), prevMasterInstance); verifyP2PMessage(DB_NAME_2, _instances.get(1), MasterSlaveSMD.States.MASTER.name(), prevMasterInstance); //re-enable the old master _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, prevMasterInstance, true); Assert.assertTrue(_clusterVerifier.verifyByPolling()); verifyP2PMessage(DB_NAME_1, prevMasterInstance, MasterSlaveSMD.States.MASTER.name(), _instances.get( 1)); verifyP2PMessage(DB_NAME_2, prevMasterInstance, MasterSlaveSMD.States.MASTER.name(), _instances.get( 1)); }
@Test public void testWithDisabledInstancesLimit() throws Exception { MaintenanceSignal maintenanceSignal = _dataAccessor.getProperty(_dataAccessor.keyBuilder().maintenance()); Assert.assertNull(maintenanceSignal); checkForRebalanceError(false); HelixAdmin admin = new ZKHelixAdmin(_gZkClient); // disable instance int i; for (i = 2; i < 2 + _maxOfflineInstancesAllowed; i++) { String instance = _participants.get(i).getInstanceName(); admin.enableInstance(CLUSTER_NAME, instance, false); } Thread.sleep(500); maintenanceSignal = _dataAccessor.getProperty(_dataAccessor.keyBuilder().maintenance()); Assert.assertNull(maintenanceSignal); String instance = _participants.get(i).getInstanceName(); admin.enableInstance(CLUSTER_NAME, instance, false); Thread.sleep(500); maintenanceSignal = _dataAccessor.getProperty(_dataAccessor.keyBuilder().maintenance()); Assert.assertNotNull(maintenanceSignal); Assert.assertNotNull(maintenanceSignal.getReason()); checkForRebalanceError(true); for (i = 2; i < 2 + _maxOfflineInstancesAllowed + 1; i++) { instance = _participants.get(i).getInstanceName(); admin.enableInstance(CLUSTER_NAME, instance, true); } admin.enableMaintenanceMode(CLUSTER_NAME, false); }
@Test (dependsOnMethods = {"testP2PStateTransitionDisabled"}) public void testP2PStateTransitionEnabledInResource() { enableP2PInCluster(CLUSTER_NAME, _configAccessor, false); enableP2PInResource(CLUSTER_NAME, _configAccessor, DB_NAME_1,true); enableP2PInResource(CLUSTER_NAME, _configAccessor, DB_NAME_2,false); // disable the master instance String prevMasterInstance = _instances.get(0); _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, prevMasterInstance, false); Assert.assertTrue(_clusterVerifier.verifyByPolling()); verifyP2PMessage(DB_NAME_1, _instances.get(1), MasterSlaveSMD.States.MASTER.name(), prevMasterInstance); verifyP2PMessage(DB_NAME_2, _instances.get(1), MasterSlaveSMD.States.MASTER.name(), _controller.getInstanceName(), 1); //re-enable the old master _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, prevMasterInstance, true); Assert.assertTrue(_clusterVerifier.verifyByPolling()); verifyP2PMessage(DB_NAME_1, prevMasterInstance, MasterSlaveSMD.States.MASTER.name(), _instances.get(1)); verifyP2PMessage(DB_NAME_2, prevMasterInstance, MasterSlaveSMD.States.MASTER.name(), _controller.getInstanceName(), 1); }
@Test public void testP2PStateTransitionDisabled() { // disable the master instance String prevMasterInstance = _instances.get(0); _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, prevMasterInstance, false); Assert.assertTrue(_clusterVerifier.verifyByPolling()); verifyP2PMessage(DB_NAME_1,_instances.get(1), MasterSlaveSMD.States.MASTER.name(), _controller.getInstanceName(), 1); verifyP2PMessage(DB_NAME_2,_instances.get(1), MasterSlaveSMD.States.MASTER.name(), _controller.getInstanceName(), 1); //re-enable the old master _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, prevMasterInstance, true); Assert.assertTrue(_clusterVerifier.verifyByPolling()); verifyP2PMessage(DB_NAME_1, prevMasterInstance, MasterSlaveSMD.States.MASTER.name(), _controller.getInstanceName(), 1); verifyP2PMessage(DB_NAME_2, prevMasterInstance, MasterSlaveSMD.States.MASTER.name(), _controller.getInstanceName(), 1); }
@Test(dependsOnMethods = {"testClusterExpansion", "testClusterExpansionByEnableInstance"}) public void testClusterShrink() throws Exception { ClusterConfig clusterConfig = _configAccessor.getClusterConfig(CLUSTER_NAME); clusterConfig.setDelayRebalaceEnabled(false); clusterConfig.setRebalanceDelayTime(0); _configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig); Assert.assertTrue(_clusterVerifier.verifyByPolling()); _migrationVerifier.reset(); _migrationVerifier.start(); // remove instance one by one for (int i = 0; i < NUM_NODE; i++) { String storageNodeName = PARTICIPANT_PREFIX + "_" + (START_PORT + i); MockParticipantManager participant = _participants.get(i); participant.syncStop(); _gSetupTool.getClusterManagementTool().enableInstance(CLUSTER_NAME, storageNodeName, false); Assert.assertTrue(_clusterVerifier.verifyByPolling()); } Assert.assertTrue(_clusterVerifier.verifyByPolling()); Assert.assertFalse(_migrationVerifier.hasLessMinActiveReplica()); Assert.assertFalse(_migrationVerifier.hasMoreReplica()); _migrationVerifier.stop(); } }
@Test public void testTaskAssignment() throws InterruptedException { _gSetupTool.getClusterManagementTool() .enableInstance(CLUSTER_NAME, PARTICIPANT_PREFIX + "_" + (_startPort + 0), false); String jobResource = TestHelper.getTestMethodName(); JobConfig.Builder jobBuilder = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND) .setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB); Workflow flow = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobResource, jobBuilder).build(); _driver.start(flow); // Wait 1 sec. The task should not be complete since it is not assigned. Thread.sleep(1000L); // The task is not assigned so the task state should be null in this case. Assert.assertNull( _driver.getJobContext(TaskUtil.getNamespacedJobName(jobResource)).getPartitionState(0)); }
@Test public void testTaskWithInstanceDisabled() throws InterruptedException { _gSetupTool.getClusterManagementTool() .enableInstance(CLUSTER_NAME, PARTICIPANT_PREFIX + "_" + (_startPort + 0), false); String jobResource = TestHelper.getTestMethodName(); JobConfig.Builder jobBuilder = new JobConfig.Builder().setCommand(MockTask.TASK_COMMAND) .setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB); Workflow flow = WorkflowGenerator.generateSingleJobWorkflowBuilder(jobResource, jobBuilder).build(); _driver.start(flow); _driver.pollForWorkflowState(jobResource, TaskState.COMPLETED); JobContext ctx = _driver.getJobContext(TaskUtil.getNamespacedJobName(jobResource)); Assert.assertEquals(ctx.getAssignedParticipant(0), PARTICIPANT_PREFIX + "_" + (_startPort + 1)); } }