@Override public List<String> get_children(String path, boolean watch) { return stateStorage.get_children(path, watch); }
@Override public List<String> backpressureTopologies() { return stateStorage.get_children(ClusterUtils.BACKPRESSURE_SUBTREE, false); }
@Override public List<String> errorTopologies() { return stateStorage.get_children(ClusterUtils.ERRORS_SUBTREE, false); }
@Override public List<String> activeStorms() { return stateStorage.get_children(ClusterUtils.STORMS_SUBTREE, false); }
@Override public List<String> activeKeys() { return stateStorage.get_children(ClusterUtils.BLOBSTORE_SUBTREE, false); }
@Override public List<String> supervisors(Runnable callback) { if (callback != null) { supervisorsCallback.set(callback); } return stateStorage.get_children(ClusterUtils.SUPERVISORS_SUBTREE, callback != null); }
@Override public List<String> blobstore(Runnable callback) { if (callback != null) { blobstoreCallback.set(callback); } stateStorage.sync_path(ClusterUtils.BLOBSTORE_SUBTREE); return stateStorage.get_children(ClusterUtils.BLOBSTORE_SUBTREE, callback != null); }
@Override public List<String> blobstoreInfo(String blobKey) { String path = ClusterUtils.blobstorePath(blobKey); stateStorage.sync_path(path); return stateStorage.get_children(path, false); }
@Override public List<NimbusSummary> nimbuses() { List<NimbusSummary> nimbusSummaries = new ArrayList<>(); List<String> nimbusIds = stateStorage.get_children(ClusterUtils.NIMBUSES_SUBTREE, false); for (String nimbusId : nimbusIds) { byte[] serialized = stateStorage.get_data(ClusterUtils.nimbusPath(nimbusId), false); // check for null which can exist because of a race condition in which nimbus nodes in zk may have been // removed when connections are reconnected after getting children in the above line if (serialized != null) { NimbusSummary nimbusSummary = ClusterUtils.maybeDeserialize(serialized, NimbusSummary.class); nimbusSummaries.add(nimbusSummary); } } return nimbusSummaries; }
@Override public Set<String> idsOfTopologiesWithPrivateWorkerKeys() { HashSet<String> ret = new HashSet<>(); for (WorkerTokenServiceType type : WorkerTokenServiceType.values()) { String path = ClusterUtils.secretKeysPath(type); try { ret.addAll(stateStorage.get_children(path, false)); } catch (RuntimeException e) { //If the node does not exist it is fine/expected... if (!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { throw e; } } } return ret; } }
@Override public void syncRemoteAssignments(Map<String, byte[]> remote) { if (null != remote) { this.assignmentsBackend.syncRemoteAssignments(remote); } else { Map<String, byte[]> tmp = new HashMap<>(); List<String> stormIds = this.stateStorage.get_children(ClusterUtils.ASSIGNMENTS_SUBTREE, false); for (String stormId : stormIds) { byte[] assignment = this.stateStorage.get_data(ClusterUtils.assignmentPath(stormId), false); tmp.put(stormId, assignment); } this.assignmentsBackend.syncRemoteAssignments(tmp); } }
@Override public long getNextPrivateWorkerKeyVersion(WorkerTokenServiceType type, String topologyId) { String path = ClusterUtils.secretKeysPath(type, topologyId); try { List<String> versions = stateStorage.get_children(path, false); return versions.stream().mapToLong(Long::valueOf).max().orElse(0); } catch (RuntimeException e) { if (Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { //If the node does not exist, then the version must be 0 return 0; } throw e; } }
@Override public List<ProfileRequest> getTopologyProfileRequests(String stormId) { List<ProfileRequest> profileRequests = new ArrayList<>(); String path = ClusterUtils.profilerConfigPath(stormId); if (stateStorage.node_exists(path, false)) { List<String> strs = stateStorage.get_children(path, false); for (String str : strs) { String childPath = path + ClusterUtils.ZK_SEPERATOR + str; byte[] raw = stateStorage.get_data(childPath, false); ProfileRequest request = ClusterUtils.maybeDeserialize(raw, ProfileRequest.class); if (request != null) { profileRequests.add(request); } } } return profileRequests; }
@Override public List<ErrorInfo> errors(String stormId, String componentId) { List<ErrorInfo> errorInfos = new ArrayList<>(); String path = ClusterUtils.errorPath(stormId, componentId); if (stateStorage.node_exists(path, false)) { List<String> childrens = stateStorage.get_children(path, false); for (String child : childrens) { String childPath = path + ClusterUtils.ZK_SEPERATOR + child; ErrorInfo errorInfo = ClusterUtils.maybeDeserialize(stateStorage.get_data(childPath, false), ErrorInfo.class); if (errorInfo != null) { errorInfos.add(errorInfo); } } } Collections.sort(errorInfos, new Comparator<ErrorInfo>() { public int compare(ErrorInfo arg0, ErrorInfo arg1) { return Integer.compare(arg1.get_error_time_secs(), arg0.get_error_time_secs()); } }); return errorInfos; }
/** * Check whether a topology is in throttle-on status or not: if the backpresure/storm-id dir is not empty, this topology has * throttle-on, otherwise throttle-off. But if the backpresure/storm-id dir is not empty and has not been updated for more than * timeoutMs, we treat it as throttle-off. This will prevent the spouts from getting stuck indefinitely if something wrong happens. * * @param stormId The topology Id * @param timeoutMs How long until the backpressure znode is invalid. * @param callback The callback function * @return True is backpresure/storm-id dir is not empty and at least one of the backpressure znodes has not timed out; false otherwise. */ @Override public boolean topologyBackpressure(String stormId, long timeoutMs, Runnable callback) { if (callback != null) { backPressureCallback.put(stormId, callback); } String path = ClusterUtils.backpressureStormRoot(stormId); long mostRecentTimestamp = 0; if (stateStorage.node_exists(path, false)) { List<String> children = stateStorage.get_children(path, callback != null); mostRecentTimestamp = children.stream() .map(childPath -> stateStorage.get_data(ClusterUtils.backpressurePath(stormId, childPath), false)) .filter(data -> data != null) .mapToLong(data -> ByteBuffer.wrap(data).getLong()) .max() .orElse(0); } boolean ret = ((System.currentTimeMillis() - mostRecentTimestamp) < timeoutMs); LOG.debug("topology backpressure is {}", ret ? "on" : "off"); return ret; }
stateStorage.create_sequential(path + ClusterUtils.ZK_SEPERATOR + "e", serData, defaultAcls); stateStorage.set_data(lastErrorPath, serData, defaultAcls); List<String> childrens = stateStorage.get_children(path, false);
@Override public void removeExpiredPrivateWorkerKeys(String topologyId) { for (WorkerTokenServiceType type : WorkerTokenServiceType.values()) { String basePath = ClusterUtils.secretKeysPath(type, topologyId); try { for (String version : stateStorage.get_children(basePath, false)) { String fullPath = basePath + ClusterUtils.ZK_SEPERATOR + version; try { PrivateWorkerKey key = ClusterUtils.maybeDeserialize(stateStorage.get_data(fullPath, false), PrivateWorkerKey.class); if (Time.currentTimeMillis() > key.get_expirationTimeMillis()) { LOG.info("Removing expired worker key {}", fullPath); stateStorage.delete_node(fullPath); } } catch (RuntimeException e) { //This should never happen because only the primary nimbus is active, but just in case // declare the race safe, even if we lose it. if (!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { throw e; } } } } catch (RuntimeException e) { //No node for basePath is OK, nothing to remove if (!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { throw e; } } } }
@Override public List<String> blobstore(Runnable callback) { if (callback != null) { blobstoreCallback.set(callback); } stateStorage.sync_path(ClusterUtils.BLOBSTORE_SUBTREE); return stateStorage.get_children(ClusterUtils.BLOBSTORE_SUBTREE, callback != null); }
@Override public List nimbuses() { List<NimbusSummary> nimbusSummaries = new ArrayList<>(); List<String> nimbusIds = stateStorage.get_children(ClusterUtils.NIMBUSES_SUBTREE, false); for (String nimbusId : nimbusIds) { byte[] serialized = stateStorage.get_data(ClusterUtils.nimbusPath(nimbusId), false); NimbusSummary nimbusSummary = ClusterUtils.maybeDeserialize(serialized, NimbusSummary.class); nimbusSummaries.add(nimbusSummary); } return nimbusSummaries; }
@Override public List<String> blobstoreInfo(String blobKey) { String path = ClusterUtils.blobstorePath(blobKey); stateStorage.sync_path(path); return stateStorage.get_children(path, false); }