private static Map<TopicPartition, OffsetAndMetadata> getOffsetsAtPath( CuratorFramework curator, ObjectMapper objectMapper, String partitionsRoot) throws Exception { Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>(); if (curator.checkExists().forPath(partitionsRoot) == null) { throw new RuntimeException("No such path " + partitionsRoot); } List<String> partitionPaths = curator.getChildren().forPath(partitionsRoot); for (String partitionPath : partitionPaths) { String absPartitionPath = partitionsRoot + "/" + partitionPath; LOG.info("Reading offset data from path {}", absPartitionPath); byte[] partitionBytes = curator.getData().forPath(absPartitionPath); Map<String, Object> partitionMetadata = objectMapper.readValue(partitionBytes, new TypeReference<Map<String, Object>>() { }); String topic = (String) partitionMetadata.get("topic"); int partition = ((Number) partitionMetadata.get("partition")).intValue(); long offset = ((Number) partitionMetadata.get("offset")).longValue(); offsets.put(new TopicPartition(topic, partition), new OffsetAndMetadata(offset)); } return offsets; }
@Override public List<String> addTargetChildListener(String path, CuratorWatcher listener) { try { return client.getChildren().usingWatcher(listener).forPath(path); } catch (NoNodeException e) { return null; } catch (Exception e) { throw new IllegalStateException(e.getMessage(), e); } }
public List<String> getChildren(CuratorFramework zk, String path, boolean watch) throws Exception { String normPath = PathUtils.normalize_path(path); if (watch) { return zk.getChildren().watched().forPath(normPath); } else { return zk.getChildren().forPath(normPath); } }
public List<String> list(String path) { path = "/" + path; try { if (_curator.checkExists().forPath(path) == null) { return new ArrayList<>(); } else { return _curator.getChildren().forPath(path); } } catch (Exception e) { throw new RuntimeException(e); } }
public List<TaskAnnouncement> getAnnouncements() throws Exception { final List<TaskAnnouncement> announcements = new ArrayList<>(); for (String id : curatorFramework.getChildren().forPath(getStatusPathForWorker())) { announcements.add( jsonMapper.readValue( curatorFramework.getData().forPath(getStatusPathForId(id)), TaskAnnouncement.class ) ); } return announcements; }
public List<String> lookupURLs() { List<String> serverHosts = new ArrayList<>(); try (CuratorFramework zooKeeperClient = CuratorFrameworkFactory.builder().connectString(zooKeeperEnsemble) .retryPolicy(new ExponentialBackoffRetry(1000, 3)).build()) { zooKeeperClient.start(); zooKeeperClient.blockUntilConnected(10, TimeUnit.SECONDS); List<String> serverNodes = zooKeeperClient.getChildren().forPath("/" + zooKeeperNamespace); for (String serverNode : serverNodes) { String serverInfo = new String( zooKeeperClient.getData().forPath("/" + zooKeeperNamespace + "/" + serverNode), StandardCharsets.UTF_8); String serverURL = constructURL(serverInfo); serverHosts.add(serverURL); } } catch (Exception e) { LOG.failedToGetZookeeperUrls(e); throw new RuntimeException(e); } return serverHosts; }
int maxRetries = hiveConf.getIntVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CONNECTION_MAX_RETRIES); CuratorFramework zooKeeperClient = CuratorFrameworkFactory.builder().connectString(zooKeeperEnsemble) .retryPolicy(new ExponentialBackoffRetry(baseSleepTime, maxRetries)).build(); zooKeeperClient.start(); List<String> znodePaths = zooKeeperClient.getChildren().forPath( ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + rootNamespace); List<String> znodePathsUpdated; LOG.warn("Will attempt to remove the znode: " + fullZnodePath + " from ZooKeeper"); System.out.println("Will attempt to remove the znode: " + fullZnodePath + " from ZooKeeper"); zooKeeperClient.delete().guaranteed().inBackground(new DeleteCallBack()) .forPath(fullZnodePath); zooKeeperClient.getChildren().forPath( ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + rootNamespace);
if (curator.checkExists().forPath(basePath) == null) { return; List<String> childPaths = curator.getChildren() .forPath(basePath); SingularityPendingRequest pendingRequest = requestTranscoder.fromBytes(curator.getData() .forPath(String.format("%s/%s", basePath, childPath))); if (pendingRequest.getPendingType() == PendingType.IMMEDIATE) { String rewrittenPath = new SingularityDeployKey(pendingRequest.getRequestId(), pendingRequest.getDeployId()) requestManager.addToPendingQueue(pendingRequest); curator.delete() .forPath(String.format("%s/%s", basePath, childPath)); rewrittenPaths += 1; } else {
private static void checkRootAcls(Configuration conf, String path, String user) { int stime = conf.getInt(ZK_DTSM_ZK_SESSION_TIMEOUT, ZK_DTSM_ZK_SESSION_TIMEOUT_DEFAULT), ctime = conf.getInt(ZK_DTSM_ZK_CONNECTION_TIMEOUT, ZK_DTSM_ZK_CONNECTION_TIMEOUT_DEFAULT); CuratorFramework zkClient = CuratorFrameworkFactory.builder().namespace(null) .retryPolicy(new RetryOneTime(10)).sessionTimeoutMs(stime).connectionTimeoutMs(ctime) .ensembleProvider(new FixedEnsembleProvider(conf.get(ZK_DTSM_ZK_CONNECTION_STRING))) .build(); // Hardcoded from a private field in ZKDelegationTokenSecretManager. // We need to check the path under what it sets for namespace, since the namespace is // created with world ACLs. String nsPath = "/" + path + "/ZKDTSMRoot"; Id currentUser = new Id("sasl", user); try { zkClient.start(); List<String> children = zkClient.getChildren().forPath(nsPath); for (String child : children) { String childPath = nsPath + "/" + child; checkAcls(zkClient, currentUser, childPath); } } catch (Exception e) { throw new RuntimeException(e); } finally { zkClient.close(); } }
private void checkRedundantNode(String node) { try { // Nothing to do if it is a lock mode if (getLockMode(node) != null) { return; } List<String> children = curatorFramework.getChildren().forPath(node); for (String child : children) { checkRedundantNode(node + "/" + child); } children = curatorFramework.getChildren().forPath(node); if ((children == null) || (children.isEmpty())) { curatorFramework.delete().forPath(node); } } catch (Exception e) { LOG.warn("Error in checkRedundantNode for node " + node, e); } }
private void determineLeadership() { ReentrantReadWriteLock.WriteLock lock = this.readWriteLock.writeLock(); lock.lock(); try { List<String> children = this.zooKeeper.getChildren().forPath(this.leaderElectionNode); Collections.sort(children); int idx = children.indexOf(this.nodeId); if (idx == 0) { Stat stat = this.zooKeeper.checkExists().forPath(this.leaderNode); if (stat == null) { this.zooKeeper.create().forPath(this.leaderNode, serializeMetadata(this.localMetadata)); } else { this.zooKeeper.setData().forPath(this.leaderNode, serializeMetadata(this.localMetadata)); } this.isLeader = true; } else { this.isLeader = false; String watchedNode = this.leaderElectionNode + "/" + children.get(idx - 1); this.zooKeeper.checkExists().usingWatcher(new DetermineLeadershipWatcher()).forPath(watchedNode); } findLeader(); } catch (KeeperException exc) { reset(); } catch (Throwable exc) { log.error("Fatal failure.", exc); this.fatalFailure = true; } finally { lock.unlock(); } }
@Test public void testLockInfo() throws Exception { final String LOCK_INFO = "This is the information in the lock"; final String key1 = LockManager.getLockKey("test_lock", "key1"); try (final LockManager lockManager = new LockManager(conf)) { assertTrue(lockManager.lock(key1, LOCK_INFO)); final LockManagerConfiguration lockConf = new LockManagerConfiguration(conf); final ZookeeperConfiguration zkConf = new ZookeeperConfiguration(conf); final CuratorFramework client = CuratorFrameworkFactory.builder() .connectString(zkConf.getZkQuorum()) .retryPolicy(new BoundedExponentialBackoffRetry(1000, 5000, 3)) .namespace(lockConf.getZkBasePath()) .sessionTimeoutMs(lockConf.getZkSessionTimeoutMs()) .connectionTimeoutMs(lockConf.getZkConnectionTimeoutMs()) .build(); client.start(); final String node = client.getChildren().forPath(key1).get(0); final String data = new String(client.getData().forPath(key1 + "/" + node)); assertEquals(LOCK_INFO, data); client.close(); lockManager.unlock(key1); } }
@Override public void applyMigration() { final long start = System.currentTimeMillis(); try { if (curator.checkExists().forPath(PENDING_TASKS_ROOT) == null) { return; } } catch (Exception e) { throw Throwables.propagate(e); } try { for (String pendingTaskId : curator.getChildren().forPath(PENDING_TASKS_ROOT)) { SingularityPendingTaskId newPendingTaskId = createFrom(pendingTaskId, start); if (!newPendingTaskId.toString().equals(pendingTaskId)) { LOG.info("Migrating {} to {}", pendingTaskId, newPendingTaskId); Optional<String> cmdLineArgs = getCmdLineArgs(pendingTaskId); taskManager.savePendingTask( new SingularityPendingTaskBuilder() .setPendingTaskId(newPendingTaskId) .setCmdLineArgsList(cmdLineArgs.isPresent() ? Optional.of(Collections.singletonList(cmdLineArgs.get())) : Optional.<List<String>> absent()) .build()); curator.delete().forPath(ZKPaths.makePath(PENDING_TASKS_ROOT, pendingTaskId)); } } } catch (Exception e) { throw Throwables.propagate(e); } }
curatorFramework.delete().forPath(res); throw new LockException("The created node does not contain a sequence number: " + res); List<String> children = curatorFramework.getChildren().forPath(lastName); curatorFramework.delete().forPath(res); } finally { if (LOG.isDebugEnabled()) { try { String data = new String(curatorFramework.getData().forPath(child)); conflictingLocks.add(data); } catch (Exception e) {
/** * @return Retrieve lists of hosts from ZooKeeper */ private List<String> retrieveHosts() { List<String> serverHosts = new ArrayList<>(); try (CuratorFramework zooKeeperClient = CuratorFrameworkFactory.builder() .connectString(getZookeeperEnsemble()) .retryPolicy(new ExponentialBackoffRetry(1000, 3)) .build()) { zooKeeperClient.start(); zooKeeperClient.blockUntilConnected(10, TimeUnit.SECONDS); List<String> serverNodes = zooKeeperClient.getChildren().forPath("/live_nodes"); for (String serverNode : serverNodes) { String serverURL = constructURL(serverNode); serverHosts.add(serverURL); } } catch (Exception e) { LOG.failedToGetZookeeperUrls(e); throw new RuntimeException(e); } return serverHosts; }
public static void clear(TestingServer testingServer) throws Exception { CuratorFramework client = CuratorFrameworkFactory.newClient( testingServer.getConnectString(), new RetryOneTime(RETRY_DELAY_MS)); client.start(); for (String rootNode : client.getChildren().forPath("/")) { if (!rootNode.equals(ZOOKEEPER_ROOT_NODE_NAME)) { client.delete().deletingChildrenIfNeeded().forPath("/" + rootNode); } } client.close(); } }
@Override public TreeData load(String key) throws Exception { Stat stat = client.checkExists().forPath(key); if (stat!= null) { byte[] bytes = dataIsCompressed ? client.getData().decompressed().usingWatcher(watcher).forPath(key) : client.getData().usingWatcher(watcher).forPath(key); List<String> children = client.getChildren().usingWatcher(watcher).forPath(key); return new TreeData(key, stat, bytes, children); } else { return null; } } });
private static void copyChildren(CuratorFramework curatorClient, String oldPath, String newPath, int depth) throws Exception { for (String childNode : curatorClient.getChildren().forPath(oldPath)) { String oldChildPath = oldPath + File.separator + childNode; Stat stat = curatorClient.checkExists().forPath(oldChildPath); // no need to check exists String newChildPath = newPath + File.separator + childNode; log.info("Creating new znode: " + newChildPath); curatorClient.create().withMode(CreateMode.PERSISTENT).withACL(OPEN_ACL_UNSAFE) .forPath(newChildPath, curatorClient.getData().storingStatIn(stat).forPath(oldChildPath)); if (--depth > 0) { copyChildren(curatorClient, oldChildPath, newChildPath, depth); } } }
public SpiderWatcher() { String connectString = "192.168.57.133:2181"; RetryPolicy retryPolicy = new ExponentialBackoffRetry(3000, 3); client = CuratorFrameworkFactory.newClient(connectString, retryPolicy); client.start(); //指定需要监控的父节点 try { //表示给spider节点注册一个监视器 children = client.getChildren().usingWatcher(this).forPath("/spider"); } catch (Exception e) { // TODO Auto-generated catch block e.printStackTrace(); } }
@Override public List<String> getChildren(String path) { try { return client.getChildren().forPath(path); } catch (NoNodeException e) { return null; } catch (Exception e) { throw new IllegalStateException(e.getMessage(), e); } }