@Override public void removeLastSequenceIds(String peerId, List<String> encodedRegionNames) throws ReplicationException { try { List<ZKUtilOp> listOfOps = encodedRegionNames.stream().map(n -> getSerialReplicationRegionPeerNode(n, peerId)) .map(ZKUtilOp::deleteNodeFailSilent).collect(Collectors.toList()); ZKUtil.multiOrSequential(zookeeper, listOfOps, true); } catch (KeeperException e) { throw new ReplicationException("Failed to remove last sequence ids, peerId=" + peerId + ", encodedRegionNames.size=" + encodedRegionNames.size(), e); } }
@Override public void removeHFileRefs(String peerId, List<String> files) throws ReplicationException { String peerNode = getHFileRefsPeerNode(peerId); LOG.debug("Removing hfile references {} from queue {}", files, peerNode); List<ZKUtilOp> listOfOps = files.stream().map(n -> getHFileNode(peerNode, n)) .map(ZKUtilOp::deleteNodeFailSilent).collect(toList()); LOG.debug("The multi list size for removing hfile references in zk for node {} is {}", peerNode, listOfOps.size()); try { ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true); } catch (KeeperException e) { throw new ReplicationException("Failed to remove hfile reference from peer " + peerId, e); } }
@Override public void addHFileRefs(String peerId, List<Pair<Path, Path>> pairs) throws ReplicationException { String peerNode = getHFileRefsPeerNode(peerId); LOG.debug("Adding hfile references {} in queue {}", pairs, peerNode); List<ZKUtilOp> listOfOps = pairs.stream().map(p -> p.getSecond().getName()) .map(n -> getHFileNode(peerNode, n)) .map(f -> ZKUtilOp.createAndFailSilent(f, HConstants.EMPTY_BYTE_ARRAY)).collect(toList()); LOG.debug("The multi list size for adding hfile references in zk for node {} is {}", peerNode, listOfOps.size()); try { ZKUtil.multiOrSequential(this.zookeeper, listOfOps, true); } catch (KeeperException e) { throw new ReplicationException("Failed to add hfile reference to peer " + peerId, e); } }
@Override public void setLastSequenceIds(String peerId, Map<String, Long> lastSeqIds) throws ReplicationException { try { // No need CAS and retry here, because it'll call setLastSequenceIds() for disabled peers // only, so no conflict happen. List<ZKUtilOp> listOfOps = new ArrayList<>(); for (Entry<String, Long> lastSeqEntry : lastSeqIds.entrySet()) { String path = getSerialReplicationRegionPeerNode(lastSeqEntry.getKey(), peerId); ZKUtil.createWithParents(zookeeper, path); listOfOps.add(ZKUtilOp.setData(path, ZKUtil.positionToByteArray(lastSeqEntry.getValue()))); } if (!listOfOps.isEmpty()) { ZKUtil.multiOrSequential(zookeeper, listOfOps, true); } } catch (KeeperException e) { throw new ReplicationException("Failed to set last sequence ids, peerId=" + peerId + ", size of lastSeqIds=" + lastSeqIds.size(), e); } }
multiOrSequential(zkw, ops, runSequentialOnMultiFailure);
@Override public void setWALPosition(ServerName serverName, String queueId, String fileName, long position, Map<String, Long> lastSeqIds) throws ReplicationException { try { for (int retry = 0;; retry++) { List<ZKUtilOp> listOfOps = new ArrayList<>(); if (position > 0) { listOfOps.add(ZKUtilOp.setData(getFileNode(serverName, queueId, fileName), ZKUtil.positionToByteArray(position))); } // Persist the max sequence id(s) of regions for serial replication atomically. addLastSeqIdsToOps(queueId, lastSeqIds, listOfOps); if (listOfOps.isEmpty()) { return; } try { ZKUtil.multiOrSequential(zookeeper, listOfOps, false); return; } catch (KeeperException.BadVersionException | KeeperException.NodeExistsException e) { LOG.warn( "Bad version(or node exist) when persist the last pushed sequence id to zookeeper storage, " + "Retry = " + retry + ", serverName=" + serverName + ", queueId=" + queueId + ", fileName=" + fileName); } } } catch (KeeperException e) { throw new ReplicationException("Failed to set log position (serverName=" + serverName + ", queueId=" + queueId + ", fileName=" + fileName + ", position=" + position + ")", e); } }
multiOrSequential(zkw, ops, runSequentialOnMultiFailure);
@Override public void transitPeerSyncReplicationState(String peerId) throws ReplicationException { String newStateNode = getNewSyncReplicationStateNode(peerId); try { byte[] data = ZKUtil.getData(zookeeper, newStateNode); ZKUtil.multiOrSequential(zookeeper, Arrays.asList(ZKUtilOp.setData(newStateNode, NONE_STATE_ZNODE_BYTES), ZKUtilOp.setData(getSyncReplicationStateNode(peerId), data)), false); } catch (KeeperException | InterruptedException e) { throw new ReplicationException( "Error transiting sync replication state for peer with id=" + peerId, e); } }
ZKUtil.multiOrSequential(watcher, zkOps, false); } catch (KeeperException e) { LOG.error("Failed to write to rsGroupZNode", e);
ops.add(ZKUtilOp.deleteNodeFailSilent(path)); try { ZKUtil.multiOrSequential(zkw, ops, false); } catch (KeeperException.NoNodeException nne) { caughtNoNode = true; ops.add(ZKUtilOp.setData(path, Bytes.toBytes(path))); try { ZKUtil.multiOrSequential(zkw, ops, false); } catch (KeeperException.NoNodeException nne) { caughtNoNode = true; ops = new LinkedList<>(); ops.add(ZKUtilOp.createAndFailSilent(path, Bytes.toBytes(path))); ZKUtil.multiOrSequential(zkw, ops, false); try { ZKUtil.multiOrSequential(zkw, ops, false); } catch (KeeperException.NodeExistsException nee) { caughtNodeExists = true;
@Test public void testSimpleMulti() throws Exception { // null multi ZKUtil.multiOrSequential(zkw, null, false); // empty multi ZKUtil.multiOrSequential(zkw, new LinkedList<>(), false); // single create String path = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSimpleMulti"); LinkedList<ZKUtilOp> singleCreate = new LinkedList<>(); singleCreate.add(ZKUtilOp.createAndFailSilent(path, new byte[0])); ZKUtil.multiOrSequential(zkw, singleCreate, false); assertTrue(ZKUtil.checkExists(zkw, path) != -1); // single setdata LinkedList<ZKUtilOp> singleSetData = new LinkedList<>(); byte [] data = Bytes.toBytes("foobar"); singleSetData.add(ZKUtilOp.setData(path, data)); ZKUtil.multiOrSequential(zkw, singleSetData, false); assertTrue(Bytes.equals(ZKUtil.getData(zkw, path), data)); // single delete LinkedList<ZKUtilOp> singleDelete = new LinkedList<>(); singleDelete.add(ZKUtilOp.deleteNodeFailSilent(path)); ZKUtil.multiOrSequential(zkw, singleDelete, false); assertTrue(ZKUtil.checkExists(zkw, path) == -1); }
ZKUtil.multiOrSequential(zookeeper, listOfOps, false);
@Override public void addPeer(String peerId, ReplicationPeerConfig peerConfig, boolean enabled, SyncReplicationState syncReplicationState) throws ReplicationException { List<ZKUtilOp> multiOps = Arrays.asList( ZKUtilOp.createAndFailSilent(getPeerNode(peerId), ReplicationPeerConfigUtil.toByteArray(peerConfig)), ZKUtilOp.createAndFailSilent(getPeerStateNode(peerId), enabled ? ENABLED_ZNODE_BYTES : DISABLED_ZNODE_BYTES), ZKUtilOp.createAndFailSilent(getSyncReplicationStateNode(peerId), SyncReplicationState.toByteArray(syncReplicationState)), ZKUtilOp.createAndFailSilent(getNewSyncReplicationStateNode(peerId), NONE_STATE_ZNODE_BYTES)); try { ZKUtil.createWithParents(zookeeper, peersZNode); ZKUtil.multiOrSequential(zookeeper, multiOps, false); } catch (KeeperException e) { throw new ReplicationException( "Could not add peer with id=" + peerId + ", peerConfig=>" + peerConfig + ", state=" + (enabled ? "ENABLED" : "DISABLED") + ", syncReplicationState=" + syncReplicationState, e); } }
@Test public void testRunSequentialOnMultiFailure() throws Exception { String path1 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "runSequential1"); String path2 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "runSequential2"); String path3 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "runSequential3"); String path4 = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "runSequential4"); // create some nodes that we will use later LinkedList<ZKUtilOp> ops = new LinkedList<>(); ops.add(ZKUtilOp.createAndFailSilent(path1, Bytes.toBytes(path1))); ops.add(ZKUtilOp.createAndFailSilent(path2, Bytes.toBytes(path2))); ZKUtil.multiOrSequential(zkw, ops, false); // test that, even with operations that fail, the ones that would pass will pass // with runSequentialOnMultiFailure ops = new LinkedList<>(); ops.add(ZKUtilOp.setData(path1, Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1)))); // pass ops.add(ZKUtilOp.deleteNodeFailSilent(path2)); // pass ops.add(ZKUtilOp.deleteNodeFailSilent(path3)); // fail -- node doesn't exist ops.add(ZKUtilOp.createAndFailSilent(path4, Bytes.add(Bytes.toBytes(path4), Bytes.toBytes(path4)))); // pass ZKUtil.multiOrSequential(zkw, ops, true); assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1), Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1)))); assertTrue(ZKUtil.checkExists(zkw, path2) == -1); assertTrue(ZKUtil.checkExists(zkw, path3) == -1); assertFalse(ZKUtil.checkExists(zkw, path4) == -1); }
@Test public void testSetDataWithVersion() throws Exception { ZKUtil.createWithParents(ZKW, "/s1/s2/s3"); int v0 = getZNodeDataVersion("/s1/s2/s3"); assertEquals(0, v0); ZKUtil.setData(ZKW, "/s1/s2/s3", Bytes.toBytes(12L)); int v1 = getZNodeDataVersion("/s1/s2/s3"); assertEquals(1, v1); ZKUtil.multiOrSequential(ZKW, ImmutableList.of(ZKUtilOp.setData("/s1/s2/s3", Bytes.toBytes(13L), v1)), false); int v2 = getZNodeDataVersion("/s1/s2/s3"); assertEquals(2, v2); }
@Test public void testSingleFailureInMulti() throws Exception { // try a multi where all but one operation succeeds String pathA = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSingleFailureInMultiA"); String pathB = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSingleFailureInMultiB"); String pathC = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "testSingleFailureInMultiC"); LinkedList<ZKUtilOp> ops = new LinkedList<>(); ops.add(ZKUtilOp.createAndFailSilent(pathA, Bytes.toBytes(pathA))); ops.add(ZKUtilOp.createAndFailSilent(pathB, Bytes.toBytes(pathB))); ops.add(ZKUtilOp.deleteNodeFailSilent(pathC)); boolean caughtNoNode = false; try { ZKUtil.multiOrSequential(zkw, ops, false); } catch (KeeperException.NoNodeException nne) { caughtNoNode = true; } assertTrue(caughtNoNode); // assert that none of the operations succeeded assertTrue(ZKUtil.checkExists(zkw, pathA) == -1); assertTrue(ZKUtil.checkExists(zkw, pathB) == -1); assertTrue(ZKUtil.checkExists(zkw, pathC) == -1); }
create4Nodes.add(ZKUtilOp.createAndFailSilent(path3, Bytes.toBytes(path3))); create4Nodes.add(ZKUtilOp.createAndFailSilent(path4, Bytes.toBytes(path4))); ZKUtil.multiOrSequential(zkw, create4Nodes, false); assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1), Bytes.toBytes(path1))); assertTrue(Bytes.equals(ZKUtil.getData(zkw, path2), Bytes.toBytes(path2))); ZKUtil.multiOrSequential(zkw, ops, false); assertTrue(Bytes.equals(ZKUtil.getData(zkw, path1), Bytes.add(Bytes.toBytes(path1), Bytes.toBytes(path1))));
ZKUtil.multiOrSequential(zkw, ops, false); ZKUtil.multiOrSequential(zkw, ops, false); } catch (KeeperException.NodeExistsException nee) { ZKUtil.multiOrSequential(zkw, ops, false); } catch (KeeperException.NoNodeException nne) {
@Override public void removeLastSequenceIds(String peerId, List<String> encodedRegionNames) throws ReplicationException { try { List<ZKUtilOp> listOfOps = encodedRegionNames.stream().map(n -> getSerialReplicationRegionPeerNode(n, peerId)) .map(ZKUtilOp::deleteNodeFailSilent).collect(Collectors.toList()); ZKUtil.multiOrSequential(zookeeper, listOfOps, true); } catch (KeeperException e) { throw new ReplicationException("Failed to remove last sequence ids, peerId=" + peerId + ", encodedRegionNames.size=" + encodedRegionNames.size(), e); } }
@Test public void testSetDataWithVersion() throws Exception { ZKUtil.createWithParents(ZKW, "/s1/s2/s3"); int v0 = getZNodeDataVersion("/s1/s2/s3"); assertEquals(0, v0); ZKUtil.setData(ZKW, "/s1/s2/s3", Bytes.toBytes(12L)); int v1 = getZNodeDataVersion("/s1/s2/s3"); assertEquals(1, v1); ZKUtil.multiOrSequential(ZKW, ImmutableList.of(ZKUtilOp.setData("/s1/s2/s3", Bytes.toBytes(13L), v1)), false); int v2 = getZNodeDataVersion("/s1/s2/s3"); assertEquals(2, v2); }