.rule(Paths.configHosts(), CREATE | DELETE, agentId) .rule(Paths.configHost(PATH_COMPONENT_WILDCARD), CREATE | DELETE, agentId) .rule(Paths.configHostId(PATH_COMPONENT_WILDCARD), CREATE | DELETE, agentId) .rule(Paths.configHostPorts(PATH_COMPONENT_WILDCARD), CREATE | DELETE, agentId) .rule(Paths.statusHosts(), CREATE | DELETE, agentId) .rule(Paths.statusHost(PATH_COMPONENT_WILDCARD), CREATE | DELETE, agentId) .rule(Paths.statusHostJobs(PATH_COMPONENT_WILDCARD), CREATE | DELETE, agentId) .rule(Paths.statusHostJob(PATH_COMPONENT_WILDCARD, PATH_COMPONENT_WILDCARD), WRITE, agentId) .rule(Paths.statusHostAgentInfo(PATH_COMPONENT_WILDCARD), WRITE, agentId) .rule(Paths.statusHostInfo(PATH_COMPONENT_WILDCARD), WRITE, agentId) .rule(Paths.statusHostLabels(PATH_COMPONENT_WILDCARD), WRITE, agentId) .rule(Paths.statusHostEnvVars(PATH_COMPONENT_WILDCARD), WRITE, agentId) .rule(Paths.statusHostUp(PATH_COMPONENT_WILDCARD), WRITE, agentId) .rule(Paths.historyJobs() + "(/.+)?", CREATE, agentId) .rule(Paths.historyJobHostEvents( PATH_COMPONENT_WILDCARD, PATH_COMPONENT_WILDCARD), DELETE, agentId) .build();
protected List<String> setupDefaultMaster(final int offset, String... args) throws Exception { curator.newNamespaceAwareEnsurePath(Paths.configHosts()).ensure(curator.getZookeeperClient()); curator.newNamespaceAwareEnsurePath(Paths.configJobs()).ensure(curator.getZookeeperClient()); curator.newNamespaceAwareEnsurePath(Paths.configJobRefs()).ensure(curator.getZookeeperClient()); curator.newNamespaceAwareEnsurePath(Paths.statusHosts()).ensure(curator.getZookeeperClient()); curator.newNamespaceAwareEnsurePath(Paths.statusMasters()).ensure(curator.getZookeeperClient()); curator.newNamespaceAwareEnsurePath(Paths.historyJobs()).ensure(curator.getZookeeperClient()); curator.newNamespaceAwareEnsurePath(Paths.configId(zkClusterId)) .ensure(curator.getZookeeperClient());
final JobId id = job.getId(); final UUID operationId = UUID.randomUUID(); final String creationPath = Paths.configJobCreation(id, operationId); final ZooKeeperClient client = provider.get("addJob"); try { try { client.ensurePath(Paths.historyJob(id)); client.transaction(create(Paths.configJob(id), job), create(Paths.configJobRefShort(id), id), create(Paths.configJobHosts(id)), create(creationPath), set(Paths.configJobs(), UUID.randomUUID().toString().getBytes())); } catch (final NodeExistsException e) { if (client.exists(creationPath) != null) {
public static void registerHost(final ZooKeeperClient client, final String idPath, final String hostname, final String hostId) throws KeeperException { log.info("registering host: {}", hostname); // This would've been nice to do in a transaction but PathChildrenCache ensures paths // so we can't know what paths already exist so assembling a suitable transaction is too // painful. client.ensurePath(Paths.configHost(hostname)); client.ensurePath(Paths.configHostJobs(hostname)); client.ensurePath(Paths.configHostPorts(hostname)); client.ensurePath(Paths.statusHost(hostname)); client.ensurePath(Paths.statusHostJobs(hostname)); // Finish registration by creating the id node last client.createAndSetData(idPath, hostId.getBytes(UTF_8)); }
@Override public boolean tryToRegister(final ZooKeeperClient client) throws KeeperException { client.ensurePath(Paths.configHosts()); client.ensurePath(Paths.configJobs()); client.ensurePath(Paths.configJobRefs()); client.ensurePath(Paths.statusHosts()); client.ensurePath(Paths.statusMasters()); client.ensurePath(Paths.historyJobs()); client.ensurePath(Paths.configDeploymentGroups()); client.ensurePath(Paths.statusDeploymentGroups()); if (upNode == null) { final String upPath = Paths.statusMasterUp(name); client.ensurePath(upPath, true); upNode = client.persistentEphemeralNode(upPath, Mode.EPHEMERAL, new byte[]{}); upNode.start(); } log.info("ZooKeeper registration complete"); return true; } }
operations.add(check(Paths.configHost(host))); final List<String> nodes = safeListRecursive(client, Paths.statusHost(host)); for (final String node : reverse(nodes)) { operations.add(delete(node)); operations.add(create(Paths.statusHost(host))); operations.add(create(Paths.statusHostJobs(host))); operations.add(delete(Paths.configHostId(host))); operations.add(create(Paths.configHostId(host), hostId.getBytes(UTF_8)));
if (client.exists(Paths.configHost(host)) == null) { throw new HostNotFoundException("host [" + host + "] does not exist"); final List<String> jobs = safeGetChildren(client, Paths.configHostJobs(host)); final String hostJobPath = Paths.configHostJob(host, job); if (client.exists(Paths.configJobHost(job, host)) != null) { operations.add(delete(Paths.configJobHost(job, host))); final List<String> history = safeListRecursive(client, Paths.historyJobHost(job, host)); for (final String s : reverse(history)) { operations.add(delete(s)); operations.add(delete(Paths.configHostJobs(host))); final List<String> nodes = safeListRecursive(client, Paths.statusHost(host)); for (final String node : reverse(nodes)) { operations.add(delete(node)); final List<String> ports = safeGetChildren(client, Paths.configHostPorts(host)); for (final String port : ports) { operations.add(delete(Paths.configHostPort(host, Integer.valueOf(port)))); operations.add(delete(Paths.configHostPorts(host))); final String idPath = Paths.configHostId(host); if (client.exists(idPath) != null) { operations.add(delete(idPath));
@Override public boolean tryToRegister(ZooKeeperClient client) throws KeeperException, HostNotFoundException { final String idPath = Paths.configHostId(name); final String hostInfoPath = Paths.statusHostInfo(name); final String upPath = Paths.statusHostUp(name); log.debug("Creating up node: {}", upPath); client.ensurePath(upPath, true);
public Map<JobId, Job> getJobs() { log.debug("getting jobs"); final String folder = Paths.configJobs(); final ZooKeeperClient client = provider.get("getJobs"); try { for (final String id : ids) { final JobId jobId = JobId.fromString(id); final String path = Paths.configJob(jobId); try { final byte[] data = client.getData(path);
@Test public void testZooKeeperClient() throws Exception { // Create the cluster ID node zk().curatorWithSuperAuth().newNamespaceAwareEnsurePath(Paths.configId(zkClusterId)) .ensure(zk().curatorWithSuperAuth().getZookeeperClient()); // We need to create a new curator because ZooKeeperClient will try to start it, // and zk().curator() has already been started. final ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3); final CuratorFramework curator = CuratorFrameworkFactory.builder() .retryPolicy(retryPolicy) .connectString(zk().connectString()) .build(); final ZooKeeperClient client = new DefaultZooKeeperClient(curator, zkClusterId); client.start(); // This should work since the cluster ID exists client.create("/test"); // Now let's remove the cluster ID client.delete(Paths.configId(zkClusterId)); // Sleep so the watcher thread in ZooKeeperClient has a chance to update state Thread.sleep(500); // Try the same operation again, and it should fail this time try { client.ensurePath(Paths.configJobs()); fail("ZooKeeper operation should have failed because cluster ID was removed"); } catch (IllegalStateException ignore) { // ignored } }
private void checkClusterIdExists(final String id, final String checker) { try { final Stat stat = client.checkExists().usingWatcher(watcher).forPath(Paths.configId(id)); final boolean exists = stat != null; clusterIdExists.set(exists); log.info("Cluster ID {} {} when checked by {}", id, exists ? "exists" : "does not exist", checker); } catch (Exception e) { clusterIdExists.set(false); log.error("Exception while checking ZooKeeper cluster ID {}", clusterId, e); } } }
public static String statusHostJob(final String host, final JobId jobId) { return statusHostJob(host, jobId.toString()); }
public static boolean isHostRegistered(final ZooKeeperClient client, final String host) { try { final Stat stat = client.exists(Paths.configHostId(host)); return stat != null; } catch (KeeperException e) { throw new HeliosRuntimeException("getting host " + host + " id failed", e); } }
private void assertHostExists(final ZooKeeperClient client, final String host) throws HostNotFoundException { try { client.getData(Paths.configHost(host)); } catch (NoNodeException e) { throw new HostNotFoundException(host, e); } catch (KeeperException e) { throw new HeliosRuntimeException(e); } }
/** * Returns a list of the hosts/agents that have been registered. */ @Override public List<String> listHosts() { try { // TODO (dano): only return hosts whose agents completed registration (i.e. has id nodes) return provider.get("listHosts").getChildren(Paths.configHosts()); } catch (KeeperException.NoNodeException e) { return emptyList(); } catch (KeeperException e) { throw new HeliosRuntimeException("listing hosts failed", e); } }
@Override void processItem(final String jobId) { final JobId id = JobId.fromString(jobId); final Job job = masterModel.getJob(id); if (job == null) { try { client.deleteRecursive(Paths.historyJob(id)); log.info("Reaped job history for job {}", jobId); } catch (NoNodeException ignored) { // Something deleted the history right before we got to it. Ignore and keep going. } catch (KeeperException e) { log.warn("error reaping job history for job {}", jobId, e); } } } }
@Override Iterable<String> collectItems() { final String path = Paths.historyJobs(); List<String> jobIds = Collections.emptyList(); try { jobIds = client.getChildren(path); } catch (KeeperException e) { log.warn("Failed to get children of znode {}", path, e); } return jobIds; }
Paths.statusHosts(), riemannFacade, TimeUnit.MINUTES, 2);
@Override public void tryToRegister(final ZooKeeperClient client) throws KeeperException { client.ensurePath(Paths.configHosts()); client.ensurePath(Paths.configJobs()); client.ensurePath(Paths.configJobRefs()); client.ensurePath(Paths.statusHosts()); client.ensurePath(Paths.statusMasters()); client.ensurePath(Paths.historyJobs()); client.ensurePath(Paths.configDeploymentGroups()); client.ensurePath(Paths.statusDeploymentGroups()); if (upNode == null) { final String upPath = Paths.statusMasterUp(name); upNode = client.persistentEphemeralNode(upPath, Mode.EPHEMERAL, new byte[]{}); upNode.start(); } log.info("ZooKeeper registration complete"); } }
if (client.exists(Paths.configHost(host)) == null) { throw new HostNotFoundException("host [" + host + "] does not exist"); final String hostJobPath = Paths.configHostJob(host, job); if (client.exists(Paths.configJobHost(job, host)) != null) { operations.add(delete(Paths.configJobHost(job, host))); final List<String> history = safeListRecursive(client, Paths.historyJobHost(job, host)); for (final String s : reverse(history)) { operations.add(delete(s)); operations.add(delete(Paths.configHostJobs(host))); final List<String> nodes = safeListRecursive(client, Paths.statusHost(host)); for (final String node : reverse(nodes)) { operations.add(delete(node)); final List<String> ports = safeGetChildren(client, Paths.configHostPorts(host)); for (final String port : ports) { operations.add(delete(Paths.configHostPort(host, Integer.valueOf(port)))); operations.add(delete(Paths.configHostPorts(host))); final String idPath = Paths.configHostId(host); if (client.exists(idPath) != null) { operations.add(delete(idPath)); operations.add(delete(Paths.configHost(host)));