public List<V> values() { List<V> values = Lists.newArrayList(); forEach((key, value) -> values.add(value)); return values; }
/** * @return a new list of all keys (makes a copy) */ public List<K> keys() { List<K> keys = Lists.newArrayList(); forEach((key, value) -> keys.add(key)); return keys; }
public int getNumberOfNamespaceBundles() { this.numberOfNamespaceBundles = 0; this.multiLayerTopicsMap.forEach((namespaceName, bundles) -> { this.numberOfNamespaceBundles += bundles.size(); }); return this.numberOfNamespaceBundles; }
private void generate() { metricsMap.forEach((key, replicationMetrics) -> { int splitPoint = key.lastIndexOf(SEPARATOR); add(key.substring(0, splitPoint), key.substring(splitPoint + 1), replicationMetrics); replicationMetrics.reset(); }); }
public Map<String, PersistentTopicStats> getTopicStats() { HashMap<String, PersistentTopicStats> stats = new HashMap<>(); topics.forEach((name, topicFuture) -> { PersistentTopic currentTopic = (PersistentTopic) topicFuture.getNow(null); if (currentTopic != null) { stats.put(name, currentTopic.getStats()); } }); return stats; }
public void checkGC(int gcIntervalInSeconds) { topics.forEach((n, t) -> { PersistentTopic topic = (PersistentTopic) t.getNow(null); if (topic != null) { topic.checkGC(gcIntervalInSeconds); } }); }
/** * Unload all the topic served by the broker service under the given service unit * * @param serviceUnit * @return */ public CompletableFuture<Integer> unloadServiceUnit(NamespaceBundle serviceUnit) { CompletableFuture<Integer> result = new CompletableFuture<Integer>(); List<CompletableFuture<Void>> closeFutures = Lists.newArrayList(); topics.forEach((name, topicFuture) -> { DestinationName topicName = DestinationName.get(name); if (serviceUnit.includes(topicName)) { // Topic needs to be unloaded log.info("[{}] Unloading topic", topicName); closeFutures.add(topicFuture.thenCompose(Topic::close)); } }); CompletableFuture<Void> aggregator = FutureUtil.waitForAll(closeFutures); aggregator.thenAccept(res -> result.complete(closeFutures.size())).exceptionally(ex -> { result.completeExceptionally(ex); return null; }); return result; }
public void checkMessageExpiry() { topics.forEach((n, t) -> { PersistentTopic topic = (PersistentTopic) t.getNow(null); if (topic != null) { topic.checkMessageExpiry(); } }); }
public CompletableFuture<Void> stopReplProducers() { List<CompletableFuture<Void>> closeFutures = Lists.newArrayList(); replicators.forEach((region, replicator) -> closeFutures.add(replicator.disconnect())); return FutureUtil.waitForAll(closeFutures); }
private synchronized CompletableFuture<Void> closeReplProducersIfNoBacklog() { List<CompletableFuture<Void>> closeFutures = Lists.newArrayList(); replicators.forEach((region, replicator) -> closeFutures.add(replicator.disconnect(true))); return FutureUtil.waitForAll(closeFutures); }
public static void generate(PulsarService pulsar, SimpleTextOutputStream stream) { String cluster = pulsar.getConfiguration().getClusterName(); AggregatedNamespaceStats namespaceStats = localNamespaceStats.get(); pulsar.getBrokerService().getMultiLayerTopicMap().forEach((namespace, bundlesMap) -> { namespaceStats.reset(); bundlesMap.forEach((bundle, topicsMap) -> { topicsMap.forEach((name, topic) -> { updateNamespaceStats(namespaceStats, topic); }); }); printNamespaceStats(stream, cluster, namespace, namespaceStats); }); }
public void monitorBacklogQuota() { topics.forEach((n, t) -> { try { PersistentTopic topic = (PersistentTopic) t.getNow(null); if (topic != null && isBacklogExceeded(topic)) { getBacklogQuotaManager().handleExceededBacklogQuota(topic); } else if (topic == null) { if (log.isDebugEnabled()) { log.debug("topic is null "); } } else { if (log.isDebugEnabled()) { log.debug("quota not exceeded for [{}]", topic.getName()); } } } catch (Exception xle) { log.warn("Backlog quota monitoring encountered :" + xle.getLocalizedMessage()); } }); }
@Override public void onUpdate(String path, Policies data, Stat stat) { final NamespaceName namespace = new NamespaceName(NamespaceBundleFactory.getNamespaceFromPoliciesPath(path)); log.info("Updated {}", path); topics.forEach((name, topicFuture) -> { if (namespace.includes(DestinationName.get(name))) { // If the topic is already created, immediately apply the updated policies, otherwise once the topic is // created it'll apply the policies update topicFuture.thenAccept(topic -> { if (log.isDebugEnabled()) { log.debug("Notifying topic that policies have changed: {}", name); } topic.onPoliciesUpdate(data); }); } }); }
public void startReplProducers() { // read repl-cluster from policies to avoid restart of replicator which are in process of disconnect and close try { Policies policies = brokerService.pulsar().getConfigurationCache().policiesCache() .get(AdminResource.path("policies", DestinationName.get(topic).getNamespace())) .orElseThrow(() -> new KeeperException.NoNodeException()); if (policies.replication_clusters != null) { Set<String> configuredClusters = Sets.newTreeSet(policies.replication_clusters); replicators.forEach((region, replicator) -> { if (configuredClusters.contains(region)) { replicator.startProducer(); } }); } } catch (Exception e) { if (log.isDebugEnabled()) { log.debug("[{}] Error getting policies while starting repl-producers {}", topic, e.getMessage()); } replicators.forEach((region, replicator) -> replicator.startProducer()); } }
@Override public void close() throws IOException { log.info("Shutting down Pulsar Broker service"); if (pulsar.getConfigurationCache() != null) { pulsar.getConfigurationCache().policiesCache().unregisterListener(this); } // unloads all namespaces gracefully without disrupting mutually unloadNamespaceBundlesGracefully(); // close replication clients replicationClients.forEach((cluster, client) -> { try { client.shutdown(); } catch (PulsarClientException e) { log.warn("Error shutting down repl client for cluster {}", cluster, e); } }); acceptorGroup.shutdownGracefully(); workerGroup.shutdownGracefully(); statsUpdater.shutdown(); inactivityMonitor.shutdown(); messageExpiryMonitor.shutdown(); backlogQuotaChecker.shutdown(); authenticationService.close(); pulsarStats.close(); log.info("Broker service completely shut down"); }
@Override public void checkMessageExpiry() { DestinationName name = DestinationName.get(topic); Policies policies; try { policies = brokerService.pulsar().getConfigurationCache().policiesCache() .get(AdminResource.path("policies", name.getNamespace())) .orElseThrow(() -> new KeeperException.NoNodeException()); if (policies.message_ttl_in_seconds != 0) { subscriptions.forEach((subName, sub) -> sub.expireMessages(policies.message_ttl_in_seconds)); replicators.forEach((region, replicator) -> replicator.expireMessages(policies.message_ttl_in_seconds)); } } catch (Exception e) { if (log.isDebugEnabled()) { log.debug("[{}] Error getting policies", topic); } } }
topic.getSubscriptions().forEach((subName, sub) -> subscriptions.add(subName)); } catch (Exception e) { log.error("[{}] Failed to get list of subscriptions for {}", clientAppId(), dn);
@Override public CompletableFuture<Void> onPoliciesUpdate(Policies data) { producers.forEach(Producer::checkPermissions); subscriptions.forEach((subName, sub) -> sub.getConsumers().forEach(Consumer::checkPermissions)); checkMessageExpiry(); return checkReplicationAndRetryOnFailure(); }
public void removeTopicFromCache(String topic) { try { DestinationName destination = DestinationName.get(topic); NamespaceBundle namespaceBundle = pulsar.getNamespaceService().getBundle(destination); checkArgument(namespaceBundle instanceof NamespaceBundle); String bundleName = namespaceBundle.toString(); String namespaceName = destination.getNamespaceObject().toString(); synchronized (multiLayerTopicsMap) { ConcurrentOpenHashMap<String, ConcurrentOpenHashMap<String, PersistentTopic>> namespaceMap = multiLayerTopicsMap .get(namespaceName); ConcurrentOpenHashMap<String, PersistentTopic> bundleMap = namespaceMap.get(bundleName); bundleMap.remove(topic); if (bundleMap.isEmpty()) { namespaceMap.remove(bundleName); } if (namespaceMap.isEmpty()) { multiLayerTopicsMap.remove(namespaceName); final ClusterReplicationMetrics clusterReplicationMetrics = pulsarStats .getClusterReplicationMetrics(); replicationClients.forEach((cluster, client) -> { clusterReplicationMetrics.remove(clusterReplicationMetrics.getKeyName(namespaceName, cluster)); }); } } } catch (Exception e) { log.warn("Got exception when retrieving bundle name during removeTopicFromCache", e); } topics.remove(topic); }
private Map<String, Map<String, PendingBookieOpsStats>> generate() throws Exception { if (pulsar.getBrokerService() != null && pulsar.getBrokerService().getTopics() != null) { pulsar.getBrokerService().getTopics().forEach((name, topicFuture) -> { PersistentTopic persistentTopic = (PersistentTopic) topicFuture.getNow(null); if (persistentTopic != null) { DestinationName destinationName = DestinationName.get(persistentTopic.getName()); put(destinationName, persistentTopic.getManagedLedger().getStats().getPendingBookieOpsStats()); } }); } return nsBookieClientStatsMap; }