/** * Extract the namespace portion out of a destination name. * * Works both with old & new convention. * * @return the namespace */ public String getNamespace() { return namespaceName.toString(); }
private boolean namespaceMatches(NamespaceName namespace, NamespaceIsolationData nsPolicyData) { for (String nsnameRegex : nsPolicyData.namespaces) { if (namespace.toString().matches(nsnameRegex)) { return true; } } return false; }
@Override public List<URL> findPrimaryBrokers(List<URL> availableBrokers, NamespaceName namespace) { if (!this.matchNamespaces(namespace.toString())) { throw new IllegalArgumentException("Namespace " + namespace.toString() + " does not match policy"); } // find the available brokers that matches primary brokers regex list return this.getMatchedBrokers(this.primary, availableBrokers); }
public static String getSLAMonitorBrokerName(ServiceUnitId ns) { Matcher m = SLA_NAMESPACE_PATTERN.matcher(ns.getNamespaceObject().toString()); if (m.matches()) { return String.format("http://%s", m.group(1)); } else { return null; } }
@Override public List<URL> findSecondaryBrokers(List<URL> availableBrokers, NamespaceName namespace) { if (!this.matchNamespaces(namespace.toString())) { throw new IllegalArgumentException("Namespace " + namespace.toString() + " does not match policy"); } // find the available brokers that matches primary brokers regex list return this.getMatchedBrokers(this.secondary, availableBrokers); }
public static String checkHeartbeatNamespace(ServiceUnitId ns) { Matcher m = HEARTBEAT_NAMESPACE_PATTERN.matcher(ns.getNamespaceObject().toString()); if (m.matches()) { LOG.debug("SLAMonitoring namespace matched the lookup namespace {}", ns.getNamespaceObject().toString()); return String.format("http://%s", m.group(1)); } else { return null; } }
@Override public int compareTo(NamespaceBundle other) { if (this.nsname.toString().compareTo(other.nsname.toString()) != 0) { return this.nsname.toString().compareTo(other.nsname.toString()); } if (equals(other)) { // completely the same range, return true return 0; } try { /** * <code>Range.insersection()</code> will throw <code>IllegalArgumentException</code> when two ranges are * not connected at all, which is a OK case for our comparison. <code>checkState</code> here is to ensure * that the two ranges we are comparing don't have overlaps. */ checkState(this.keyRange.intersection(other.keyRange).isEmpty(), "Can't compare two key ranges with non-empty intersection set"); } catch (IllegalArgumentException iae) { // OK if two ranges are not connected at all } catch (IllegalStateException ise) { // It is not OK if the intersection is not empty throw new IllegalArgumentException(ise.getMessage(), ise); } // Now we are sure that two bundles don't have overlap return this.keyRange.lowerEndpoint().compareTo(other.keyRange.lowerEndpoint()); }
@POST @Path("/{property}/{cluster}/{namespace}/{bundle}/unsubscribe/{subscription}") @ApiOperation(value = "Unsubscribes the given subscription on all destinations on a namespace bundle.") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void unsubscribeNamespaceBundle(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("subscription") String subscription, @PathParam("bundle") String bundleRange, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateAdminAccessOnProperty(property); Policies policies = getNamespacePolicies(property, cluster, namespace); if (!cluster.equals(Namespaces.GLOBAL_CLUSTER)) { validateClusterOwnership(cluster); validateClusterForProperty(property, cluster); } NamespaceName nsName = new NamespaceName(property, cluster, namespace); validateNamespaceBundleOwnership(nsName, policies.bundles, bundleRange, authoritative, true); unsubscribe(nsName, bundleRange, subscription); log.info("[{}] Successfully unsubscribed {} on namespace bundle {}/{}", clientAppId(), subscription, nsName.toString(), bundleRange); }
@POST @Path("/{property}/{cluster}/{namespace}/{bundle}/clearBacklog/{subscription}") @ApiOperation(value = "Clear backlog for a given subscription on all destinations on a namespace bundle.") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void clearNamespaceBundleBacklogForSubscription(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("subscription") String subscription, @PathParam("bundle") String bundleRange, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateAdminAccessOnProperty(property); Policies policies = getNamespacePolicies(property, cluster, namespace); if (!cluster.equals(Namespaces.GLOBAL_CLUSTER)) { validateClusterOwnership(cluster); validateClusterForProperty(property, cluster); } NamespaceName nsName = new NamespaceName(property, cluster, namespace); validateNamespaceBundleOwnership(nsName, policies.bundles, bundleRange, authoritative, true); clearBacklog(nsName, bundleRange, subscription); log.info("[{}] Successfully cleared backlog for subscription {} on namespace bundle {}/{}", clientAppId(), subscription, nsName.toString(), bundleRange); }
@POST @Path("/{property}/{cluster}/{namespace}/{bundle}/clearBacklog") @ApiOperation(value = "Clear backlog for all destinations on a namespace bundle.") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Namespace does not exist") }) public void clearNamespaceBundleBacklog(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("bundle") String bundleRange, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { validateAdminAccessOnProperty(property); Policies policies = getNamespacePolicies(property, cluster, namespace); if (!cluster.equals(Namespaces.GLOBAL_CLUSTER)) { validateClusterOwnership(cluster); validateClusterForProperty(property, cluster); } NamespaceName nsName = new NamespaceName(property, cluster, namespace); validateNamespaceBundleOwnership(nsName, policies.bundles, bundleRange, authoritative, true); clearBacklog(nsName, bundleRange, null); log.info("[{}] Successfully cleared backlog on namespace bundle {}/{}", clientAppId(), nsName.toString(), bundleRange); }
protected NamespaceBundle validateNamespaceBundleOwnership(NamespaceName fqnn, BundlesData bundles, String bundleRange, boolean authoritative, boolean readOnly) { try { NamespaceBundle nsBundle = validateNamespaceBundleRange(fqnn, bundles, bundleRange); validateBundleOwnership(nsBundle, authoritative, readOnly); return nsBundle; } catch (WebApplicationException wae) { throw wae; } catch (Exception e) { log.error("[{}] Failed to validate namespace bundle {}/{}", clientAppId(), fqnn.toString(), bundleRange, e); throw new RestException(e); } }
private void clearBacklog(NamespaceName nsName, String bundleRange, String subscription) { try { List<PersistentTopic> topicList = pulsar().getBrokerService() .getAllTopicsFromNamespaceBundle(nsName.toString(), nsName.toString() + "/" + bundleRange); List<CompletableFuture<Void>> futures = Lists.newArrayList(); if (subscription != null) { if (subscription.startsWith(pulsar().getConfiguration().getReplicatorPrefix())) { subscription = PersistentReplicator.getRemoteCluster(subscription); } for (PersistentTopic topic : topicList) { futures.add(topic.clearBacklog(subscription)); } } else { for (PersistentTopic topic : topicList) { futures.add(topic.clearBacklog()); } } FutureUtil.waitForAll(futures).get(); } catch (Exception e) { log.error("[{}] Failed to clear backlog for namespace {}/{}, subscription: {}", clientAppId(), nsName.toString(), bundleRange, subscription, e); throw new RestException(e); } }
public void refreshTopicToStatsMaps(NamespaceBundle oldBundle) { checkNotNull(oldBundle); try { // retrieve all topics under existing old bundle List<PersistentTopic> topics = getAllTopicsFromNamespaceBundle(oldBundle.getNamespaceObject().toString(), oldBundle.toString()); if (!isEmpty(topics)) { // add topic under new split bundles which already updated into NamespaceBundleFactory.bundleCache topics.stream().forEach(t -> { addTopicToStatsMaps(DestinationName.get(t.getName()), t); }); // remove old bundle from the map synchronized (multiLayerTopicsMap) { multiLayerTopicsMap.get(oldBundle.getNamespaceObject().toString()).remove(oldBundle.toString()); } } } catch (Exception e) { log.warn("Got exception while refreshing topicStats map", e); } }
@PUT @Path("/{property}/{cluster}/{namespace}/{bundle}/split") @ApiOperation(value = "Split a namespace bundle") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission") }) public void splitNamespaceBundle(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("bundle") String bundleRange, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { log.info("[{}] Split namespace bundle {}/{}/{}/{}", clientAppId(), property, cluster, namespace, bundleRange); validateSuperUserAccess(); Policies policies = getNamespacePolicies(property, cluster, namespace); if (!cluster.equals(Namespaces.GLOBAL_CLUSTER)) { validateClusterOwnership(cluster); validateClusterForProperty(property, cluster); } NamespaceName fqnn = new NamespaceName(property, cluster, namespace); validatePoliciesReadOnlyAccess(); NamespaceBundle nsBundle = validateNamespaceBundleOwnership(fqnn, policies.bundles, bundleRange, authoritative, true); try { pulsar().getNamespaceService().splitAndOwnBundle(nsBundle).get(); log.info("[{}] Successfully split namespace bundle {}", clientAppId(), nsBundle.toString()); } catch (Exception e) { log.error("[{}] Failed to split namespace bundle {}/{}", clientAppId(), fqnn.toString(), bundleRange, e); throw new RestException(e); } }
private void unsubscribe(NamespaceName nsName, String bundleRange, String subscription) { try { List<PersistentTopic> topicList = pulsar().getBrokerService() .getAllTopicsFromNamespaceBundle(nsName.toString(), nsName.toString() + "/" + bundleRange); List<CompletableFuture<Void>> futures = Lists.newArrayList(); if (subscription.startsWith(pulsar().getConfiguration().getReplicatorPrefix())) { throw new RestException(Status.PRECONDITION_FAILED, "Cannot unsubscribe a replication cursor"); } else { for (PersistentTopic topic : topicList) { PersistentSubscription sub = topic.getPersistentSubscription(subscription); if (sub != null) { futures.add(sub.delete()); } } } FutureUtil.waitForAll(futures).get(); } catch (RestException re) { throw re; } catch (Exception e) { log.error("[{}] Failed to unsubscribe {} for namespace {}/{}", clientAppId(), subscription, nsName.toString(), bundleRange, e); if (e.getCause() instanceof SubscriptionBusyException) { throw new RestException(Status.PRECONDITION_FAILED, "Subscription has active connected consumers"); } throw new RestException(e.getCause()); } }
@PUT @Path("/{property}/{cluster}/{namespace}/{bundle}/unload") @ApiOperation(value = "Unload a namespace bundle") @ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission") }) public void unloadNamespaceBundle(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("bundle") String bundleRange, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) { log.info("[{}] Unloading namespace bundle {}/{}/{}/{}", clientAppId(), property, cluster, namespace, bundleRange); validateSuperUserAccess(); Policies policies = getNamespacePolicies(property, cluster, namespace); if (!cluster.equals(Namespaces.GLOBAL_CLUSTER)) { validateClusterOwnership(cluster); validateClusterForProperty(property, cluster); } NamespaceName fqnn = new NamespaceName(property, cluster, namespace); validatePoliciesReadOnlyAccess(); NamespaceBundle nsBundle = validateNamespaceBundleOwnership(fqnn, policies.bundles, bundleRange, authoritative, true); try { pulsar().getNamespaceService().unloadNamespaceBundle(nsBundle); log.info("[{}] Successfully unloaded namespace bundle {}", clientAppId(), nsBundle.toString()); } catch (Exception e) { log.error("[{}] Failed to unload namespace bundle {}/{}", clientAppId(), fqnn.toString(), bundleRange, e); throw new RestException(e); } }
protected NamespaceBundle validateNamespaceBundleRange(NamespaceName fqnn, BundlesData bundles, String bundleRange) { try { checkArgument(bundleRange.contains("_"), "Invalid bundle range"); String[] boundaries = bundleRange.split("_"); Long lowerEndpoint = Long.decode(boundaries[0]); Long upperEndpoint = Long.decode(boundaries[1]); Range<Long> hashRange = Range.range(lowerEndpoint, BoundType.CLOSED, upperEndpoint, (upperEndpoint.equals(NamespaceBundles.FULL_UPPER_BOUND)) ? BoundType.CLOSED : BoundType.OPEN); NamespaceBundle nsBundle = pulsar().getNamespaceService().getNamespaceBundleFactory().getBundle(fqnn, hashRange); NamespaceBundles nsBundles = pulsar().getNamespaceService().getNamespaceBundleFactory().getBundles(fqnn, bundles); nsBundles.validateBundle(nsBundle); return nsBundle; } catch (Exception e) { log.error("[{}] Failed to validate namespace bundle {}/{}", clientAppId(), fqnn.toString(), bundleRange, e); throw new RestException(e); } }
public NamespaceBundleFactory(PulsarService pulsar, HashFunction hashFunc) { this.hashFunc = hashFunc; this.bundlesCache = Caffeine.newBuilder().buildAsync((NamespaceName namespace, Executor executor) -> { String path = AdminResource.joinPath(LOCAL_POLICIES_ROOT, namespace.toString()); if (LOG.isDebugEnabled()) { LOG.debug("Loading cache with bundles for {}", namespace); } if (pulsar == null || pulsar.getConfigurationCache() == null) { return CompletableFuture.completedFuture(getBundles(namespace, null)); } CompletableFuture<NamespaceBundles> future = new CompletableFuture<>(); // Read the static bundle data from the policies pulsar.getLocalZkCacheService().policiesCache().getAsync(path).thenAccept(policies -> { // If no policies defined for namespace, assume 1 single bundle BundlesData bundlesData = policies.map(p -> p.bundles).orElse(null); NamespaceBundles namespaceBundles = getBundles(namespace, bundlesData); future.complete(namespaceBundles); }).exceptionally(ex -> { future.completeExceptionally(ex); return null; }); return future; }); if (pulsar != null && pulsar.getConfigurationCache() != null) { pulsar.getLocalZkCacheService().policiesCache().registerListener(this); } this.pulsar = pulsar; }
/** * update new bundle-range to LocalZk (create a new node if not present) * * @param nsname * @param nsBundles * @param callback * @throws Exception */ private void updateNamespaceBundles(NamespaceName nsname, NamespaceBundles nsBundles, StatCallback callback) throws Exception { checkNotNull(nsname); checkNotNull(nsBundles); String path = joinPath(LOCAL_POLICIES_ROOT, nsname.toString()); Optional<LocalPolicies> policies = pulsar.getLocalZkCacheService().policiesCache().get(path); if (!policies.isPresent()) { // if policies is not present into localZk then create new policies this.pulsar.getLocalZkCacheService().createPolicies(path, false).get(cacheTimeOutInSec, SECONDS); policies = this.pulsar.getLocalZkCacheService().policiesCache().get(path); } policies.get().bundles = getBundlesData(nsBundles); this.pulsar.getLocalZkCache().getZooKeeper().setData(path, ObjectMapperFactory.getThreadLocal().writeValueAsBytes(policies.get()), -1, callback, null); }
public void removeTopicFromCache(String topic) { try { DestinationName destination = DestinationName.get(topic); NamespaceBundle namespaceBundle = pulsar.getNamespaceService().getBundle(destination); checkArgument(namespaceBundle instanceof NamespaceBundle); String bundleName = namespaceBundle.toString(); String namespaceName = destination.getNamespaceObject().toString(); synchronized (multiLayerTopicsMap) { ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, PersistentTopic>> namespaceMap = multiLayerTopicsMap .get(namespaceName); ConcurrentOpenHashMap<String, PersistentTopic> bundleMap = namespaceMap.get(bundleName); bundleMap.remove(topic); if (bundleMap.isEmpty()) { namespaceMap.remove(bundleName); } if (namespaceMap.isEmpty()) { multiLayerTopicsMap.remove(namespaceName); final ClusterReplicationMetrics clusterReplicationMetrics = pulsarStats .getClusterReplicationMetrics(); replicationClients.forEach((cluster, client) -> { clusterReplicationMetrics.remove(clusterReplicationMetrics.getKeyName(namespaceName, cluster)); }); } } } catch (Exception e) { log.warn("Got exception when retrieving bundle name during removeTopicFromCache", e); } topics.remove(topic); }