private List<HostStoreInfo> mapInstancesToHostStoreInfo( final Collection<StreamsMetadata> metadatas) { return metadatas.stream().map(metadata -> new HostStoreInfo(metadata.host(), metadata.port(), metadata.stateStoreNames())) .collect(Collectors.toList()); }
private <K> StreamsMetadata getStreamsMetadataForKey(final String storeName, final K key, final StreamPartitioner<? super K, ?> partitioner, final SourceTopicsInfo sourceTopicsInfo) { final Integer partition = partitioner.partition(sourceTopicsInfo.topicWithMostPartitions, key, null, sourceTopicsInfo.maxPartitions); final Set<TopicPartition> matchingPartitions = new HashSet<>(); for (final String sourceTopic : sourceTopicsInfo.sourceTopics) { matchingPartitions.add(new TopicPartition(sourceTopic, partition)); } for (final StreamsMetadata streamsMetadata : allMetadata) { final Set<String> stateStoreNames = streamsMetadata.stateStoreNames(); final Set<TopicPartition> topicPartitions = new HashSet<>(streamsMetadata.topicPartitions()); topicPartitions.retainAll(matchingPartitions); if (stateStoreNames.contains(storeName) && !topicPartitions.isEmpty()) { return streamsMetadata; } } return null; }
/** * Find all of the {@link StreamsMetadata}s for a given storeName * * @param storeName the storeName to find metadata for * @return A collection of {@link StreamsMetadata} that have the provided storeName */ public synchronized Collection<StreamsMetadata> getAllMetadataForStore(final String storeName) { Objects.requireNonNull(storeName, "storeName cannot be null"); if (!isInitialized()) { return Collections.emptyList(); } if (globalStores.contains(storeName)) { return allMetadata; } final List<String> sourceTopics = builder.stateStoreNameToSourceTopics().get(storeName); if (sourceTopics == null) { return Collections.emptyList(); } final ArrayList<StreamsMetadata> results = new ArrayList<>(); for (final StreamsMetadata metadata : allMetadata) { if (metadata.stateStoreNames().contains(storeName)) { results.add(metadata); } } return results; }
TestUtils.waitForCondition(() -> !StreamsMetadata.NOT_AVAILABLE.equals(streams.allMetadataForStore(KafkaMusicExample.TOP_FIVE_SONGS_STORE)), MAX_WAIT_MS, "StreamsMetadata should be available");
private void rebuildMetadata(final Map<HostInfo, Set<TopicPartition>> currentState) { allMetadata.clear(); if (currentState.isEmpty()) { return; } final Map<String, List<String>> stores = builder.stateStoreNameToSourceTopics(); for (final Map.Entry<HostInfo, Set<TopicPartition>> entry : currentState.entrySet()) { final HostInfo key = entry.getKey(); final Set<TopicPartition> partitionsForHost = new HashSet<>(entry.getValue()); final Set<String> storesOnHost = new HashSet<>(); for (final Map.Entry<String, List<String>> storeTopicEntry : stores.entrySet()) { final List<String> topicsForStore = storeTopicEntry.getValue(); if (hasPartitionsForAnyTopics(topicsForStore, partitionsForHost)) { storesOnHost.add(storeTopicEntry.getKey()); } } storesOnHost.addAll(globalStores); final StreamsMetadata metadata = new StreamsMetadata(key, storesOnHost, partitionsForHost); allMetadata.add(metadata); if (key.equals(thisHost)) { myMetadata = metadata; } } }
/** * Gets the {@link HostInfo} where the provided store and key are hosted on. This may not be the * current host that is running the application. Kafka Streams will look through all the consumer instances * under the same application id and retrieves the proper host. * * Note that the end user applications must provide `applicaiton.server` as a configuration property * for all the application instances when calling this method. If this is not available, then null maybe returned. * * @param <K> generic type for key * @param store store name * @param key key to look for * @param serializer {@link Serializer} for the key * @return the {@link HostInfo} where the key for the provided store is hosted currently */ public <K> HostInfo getHostInfo(String store, K key, Serializer<K> serializer) { StreamsMetadata streamsMetadata = this.kafkaStreamsRegistry.getKafkaStreams() .stream() .map((k) -> Optional.ofNullable(k.metadataForKey(store, key, serializer))) .filter(Optional::isPresent) .map(Optional::get) .findFirst() .orElse(null); return streamsMetadata != null ? streamsMetadata.hostInfo() : null; } }
TestUtils.waitForCondition(() -> !StreamsMetadata.NOT_AVAILABLE.equals(streams.allMetadataForStore(KafkaMusicExample.TOP_FIVE_SONGS_STORE)), MAX_WAIT_MS, "StreamsMetadata should be available");
/** * Find the metadata for the instance of this Kafka Streams Application that has the given * store and would have the given key if it exists. * @param store Store to find * @param key The key to find * @return {@link HostStoreInfo} */ public <K> HostStoreInfo streamsMetadataForStoreAndKey(final String store, final K key, final Serializer<K> serializer) { // Get metadata for the instances of this Kafka Streams application hosting the store and // potentially the value for key final StreamsMetadata metadata = streams.metadataForKey(store, key, serializer); if (metadata == null) { throw new NotFoundException(); } return new HostStoreInfo(metadata.host(), metadata.port(), metadata.stateStoreNames()); }