localTopologyManager0.expectTopologyUpdate(CacheTopology.Phase.READ_NEW_WRITE_ALL).unblock(); localTopologyManager2.expectTopologyUpdate(CacheTopology.Phase.READ_NEW_WRITE_ALL).unblock(); BlockingLocalTopologyManager.BlockedConfirmation blockedConfirmation0 = localTopologyManager0.expectPhaseConfirmation(); blockedTopology0.unblock(); blockedTopology2.unblock(); localTopologyManager0.expectPhaseConfirmation().unblock(); localTopologyManager2.expectPhaseConfirmation().unblock();
/** * x-site state transfer is triggered during a cache topology change. */ private void doXSiteStateTransferDuringTopologyChange(TopologyEvent event) throws Exception { log.debugf("Start topology change during x-site state transfer with %s", event); initBeforeTest(); final TestCaches<Object, Object> testCaches = createTestCache(event, LON); log.debugf("Controlled cache=%s, Coordinator cache=%s, Cache to remove=%s", addressOf(testCaches.controllerCache), addressOf(testCaches.coordinator), testCaches.removeIndex < 0 ? "NONE" : addressOf(cache(LON, testCaches.removeIndex))); BlockingLocalTopologyManager topologyManager = replaceTopologyManagerDefaultCache(testCaches.controllerCache.getCacheManager()); final Future<Void> topologyEventFuture = triggerTopologyChange(LON, testCaches.removeIndex); // We could get either the NO_REBALANCE update or the READ_OLD rebalance start first BlockingLocalTopologyManager.BlockedTopology blockedTopology = topologyManager.expectTopologyUpdate(); log.debug("Start x-site state transfer"); startStateTransfer(testCaches.coordinator, NYC); assertOnline(LON, NYC); blockedTopology.unblock(); topologyEventFuture.get(); awaitLocalStateTransfer(LON); awaitXSiteStateSent(LON); assertData(); }
private void doTest(Operation operation, boolean init) throws Exception { final MagicKey key = new MagicKey(cache(1), cache(2)); if (init) { cache(0).put(key, "v1"); assertInAllCache(key, "v1"); } BlockingLocalTopologyManager bltm0 = replaceTopologyManagerDefaultCache(manager(0)); BlockingLocalTopologyManager bltm1 = replaceTopologyManagerDefaultCache(manager(1)); killMember(2, null, false); //CH_UPDATE + REBALANCE_START + CH_UPDATE(blocked) bltm0.expectRebalanceStartAfterLeave().unblock(); bltm1.expectRebalanceStartAfterLeave().unblock(); bltm0.expectPhaseConfirmation().unblock(); bltm1.expectPhaseConfirmation().unblock(); //check if we are in the correct state LocalizedCacheTopology cacheTopology = TestingUtil.extractComponent(cache(0), DistributionManager.class).getCacheTopology(); DistributionInfo distributionInfo = cacheTopology.getDistribution(key); assertFalse(distributionInfo.isReadOwner()); assertTrue(distributionInfo.isWriteOwner()); assertEquals(address(1), distributionInfo.primary()); operation.put(key, "v2", cache(1)); BlockingLocalTopologyManager.finishRebalance(CacheTopology.Phase.READ_ALL_WRITE_ALL, bltm0, bltm1); waitForClusterToForm(); //let the cluster finish the state transfer assertInAllCache(key, "v2"); }
private BlockingLocalTopologyManager.BlockedTopology blockNewRebalance(BlockingLocalTopologyManager ltm, BlockingLocalTopologyManager.BlockedTopology blockedTopology) throws InterruptedException { if (blockedTopology.getCacheTopology().getPhase() == CacheTopology.Phase.READ_NEW_WRITE_ALL) { // Block the rebalance start first, and only then unblock the previous topology // Otherwise the order of the rebalance start and topology confirmation wouldn't be deterministic BlockingLocalTopologyManager.BlockedTopology newTopology = ltm.expectTopologyUpdate(CacheTopology.Phase.READ_OLD_WRITE_ALL); blockedTopology.unblock(); ltm.expectPhaseConfirmation().unblock(); return newTopology; } else { assertEquals(CacheTopology.Phase.READ_OLD_WRITE_ALL, blockedTopology.getCacheTopology().getPhase()); ltm.confirmTopologyUpdate(CacheTopology.Phase.READ_NEW_WRITE_ALL); return blockedTopology; } }
public BlockedTopology expectRebalanceStartAfterLeave() throws InterruptedException { // After a node leaves, the coordinator sends a NO_REBALANCE topology update with the remaining members // Then it immediately starts the rebalance with a READ_OLD topology update // The members can receive the topology updates in either order BlockedTopology topology0 = expectTopologyUpdate(); if (topology0.getType() == Type.REBALANCE_START) { expectTopologyUpdate(CacheTopology.Phase.NO_REBALANCE).unblock(); } else { topology0.unblock(); topology0 = expectTopologyUpdate(CacheTopology.Phase.READ_OLD_WRITE_ALL); } return topology0; }
/** * Expect a topology updates and unblock it. * <p> * If the update requires confirmation, unblock the confirmation as well. */ public void confirmTopologyUpdate(CacheTopology.Phase phase) throws InterruptedException { expectTopologyUpdate(phase).unblock(); if (needConfirmation(phase)) { expectPhaseConfirmation().unblock(); } }
public static void confirmTopologyUpdate(CacheTopology.Phase phase, BlockingLocalTopologyManager... topologyManagers) throws InterruptedException { for (BlockingLocalTopologyManager topologyManager : topologyManagers) { topologyManager.expectTopologyUpdate(phase).unblock(); } if (needConfirmation(phase)) { for (BlockingLocalTopologyManager topologyManager : topologyManagers) { topologyManager.expectPhaseConfirmation().unblock(); } } }