Optional<String> configStoreUri = getConfigStoreUri(properties); if (!configStoreUri.isPresent()) { return; Path whiteListTagUri = PathUtils.mergePaths(new Path(configStoreUri.get()), new Path(properties.getProperty(GOBBLIN_CONFIG_TAGS_WHITELIST))); getTopicsURIFromConfigStore(configClient, whiteListTagUri, filterString, runtimeConfig).stream() .filter((URI u) -> ConfigUtils.getBoolean(getConfig(configClient, u, runtimeConfig), _whitelistTopicKey, false)) .forEach(((URI u) -> whitelist.add(getTopicNameFromURI(u)))); } else if (properties.containsKey(GOBBLIN_CONFIG_TAGS_BLACKLIST)) { Preconditions.checkArgument(properties.containsKey(GOBBLIN_CONFIG_FILTER), Path blackListTagUri = PathUtils.mergePaths(new Path(configStoreUri.get()), new Path(properties.getProperty(GOBBLIN_CONFIG_TAGS_BLACKLIST))); getTopicsURIFromConfigStore(configClient, blackListTagUri, filterString, runtimeConfig).stream() .filter((URI u) -> ConfigUtils.getBoolean(getConfig(configClient, u, runtimeConfig), _blacklistTopicKey, false)) .forEach(((URI u) -> blacklist.add(getTopicNameFromURI(u)))); } else { log.warn("None of the blacklist or whitelist tags are provided");
Timer.Context context = this.metricContext.timer(CONFIG_FOR_TOPIC_TIMER).time(); configForTopic = ConfigStoreUtils.getConfigForTopic(this.props.getProperties(), KafkaSource.TOPIC_NAME, this.configClient); context.close();
/** * If config store is enabled, then intersection of topics from blacklisting/whitelisting will be taken against * the topics from config-store */ private List<KafkaTopic> getFilteredTopics(SourceState state) { List<Pattern> blacklist = DatasetFilterUtils.getPatternList(state, TOPIC_BLACKLIST); List<Pattern> whitelist = DatasetFilterUtils.getPatternList(state, TOPIC_WHITELIST); List<KafkaTopic> topics = this.kafkaConsumerClient.get().getFilteredTopics(blacklist, whitelist); Optional<String> configStoreUri = ConfigStoreUtils.getConfigStoreUri(state.getProperties()); if (configStoreUri.isPresent()) { List<KafkaTopic> topicsFromConfigStore = ConfigStoreUtils .getTopicsFromConfigStore(state.getProperties(), configStoreUri.get(), this.kafkaConsumerClient.get()); return topics.stream().filter((KafkaTopic p) -> (topicsFromConfigStore.stream() .anyMatch((KafkaTopic q) -> q.getName().equalsIgnoreCase(p.getName())))).collect(toList()); } return topics; }
public static Optional<Config> getConfigForTopic(Properties properties, String topicKey, ConfigClient configClient) { Optional<String> configStoreUri = getConfigStoreUri(properties); Optional<Config> config = Optional.<Config>absent(); if (!configStoreUri.isPresent()) { return config; } try { Preconditions.checkArgument(properties.containsKey(GOBBLIN_CONFIG_COMMONPATH), "Missing required property " + GOBBLIN_CONFIG_COMMONPATH); Preconditions.checkArgument(properties.containsKey(topicKey), "Missing required property " + topicKey); String topicName = properties.getProperty(topicKey); String commonPath = properties.getProperty(GOBBLIN_CONFIG_COMMONPATH); config = Optional.fromNullable( getConfig(configClient, getUriStringForTopic(topicName, commonPath, configStoreUri.get()), ConfigClientUtils.getOptionalRuntimeConfig(properties))); } catch (URISyntaxException e) { log.error("Unable to get config", e); } return config; }
new Path(properties.getProperty(GOBBLIN_CONFIG_TAGS_WHITELIST))); List<String> whitelistedTopics = new ArrayList<>(); ConfigStoreUtils.getTopicsURIFromConfigStore(configClient, whiteListTagUri, filterString, runtimeConfig) .stream() .filter((URI u) -> ConfigUtils.getBoolean(ConfigStoreUtils.getConfig(configClient, u, runtimeConfig), KafkaSource.TOPIC_WHITELIST, false)) .forEach(((URI u) -> whitelistedTopics.add(ConfigStoreUtils.getTopicNameFromURI(u)))); new Path(properties.getProperty(GOBBLIN_CONFIG_TAGS_BLACKLIST))); List<String> blacklistedTopics = new ArrayList<>(); ConfigStoreUtils.getTopicsURIFromConfigStore(configClient, blackListTagUri, filterString, runtimeConfig) .stream() .filter((URI u) -> ConfigUtils.getBoolean(ConfigStoreUtils.getConfig(configClient, u, runtimeConfig), KafkaSource.TOPIC_BLACKLIST, false)) .forEach(((URI u) -> blacklistedTopics.add(ConfigStoreUtils.getTopicNameFromURI(u)))); return allTopics.stream() .filter((KafkaTopic p) -> !blacklistedTopics.contains(p.getName()))
private void setTopicsFromConfigStore(State state) { Set<String> blacklistTopicsFromConfigStore = new HashSet<>(); Set<String> whitelistTopicsFromConfigStore = new HashSet<>(); ConfigStoreUtils.setTopicsFromConfigStore(state.getProperties(), blacklistTopicsFromConfigStore, whitelistTopicsFromConfigStore, MRCompactor.COMPACTION_BLACKLIST, MRCompactor.COMPACTION_WHITELIST); this.blacklist.addAll(DatasetFilterUtils.getPatternsFromStrings(new ArrayList<>(blacklistTopicsFromConfigStore))); this.whitelist.addAll(DatasetFilterUtils.getPatternsFromStrings(new ArrayList<>(whitelistTopicsFromConfigStore))); }
public static Optional<Config> getConfigForTopic(Properties properties, String topicKey, ConfigClient configClient) { Optional<String> configStoreUri = getConfigStoreUri(properties); Optional<Config> config = Optional.<Config>absent(); if (!configStoreUri.isPresent()) { return config; } try { Preconditions.checkArgument(properties.containsKey(GOBBLIN_CONFIG_COMMONPATH), "Missing required property " + GOBBLIN_CONFIG_COMMONPATH); Preconditions.checkArgument(properties.containsKey(topicKey), "Missing required property " + topicKey); String topicName = properties.getProperty(topicKey); String commonPath = properties.getProperty(GOBBLIN_CONFIG_COMMONPATH); config = Optional.fromNullable( getConfig(configClient, getUriStringForTopic(topicName, commonPath, configStoreUri.get()), ConfigClientUtils.getOptionalRuntimeConfig(properties))); } catch (URISyntaxException e) { log.error("Unable to get config", e); } return config; }
new Path(properties.getProperty(GOBBLIN_CONFIG_TAGS_WHITELIST))); List<String> whitelistedTopics = new ArrayList<>(); ConfigStoreUtils.getTopicsURIFromConfigStore(configClient, whiteListTagUri, filterString, runtimeConfig) .stream() .filter((URI u) -> ConfigUtils.getBoolean(ConfigStoreUtils.getConfig(configClient, u, runtimeConfig), KafkaSource.TOPIC_WHITELIST, false)) .forEach(((URI u) -> whitelistedTopics.add(ConfigStoreUtils.getTopicNameFromURI(u)))); new Path(properties.getProperty(GOBBLIN_CONFIG_TAGS_BLACKLIST))); List<String> blacklistedTopics = new ArrayList<>(); ConfigStoreUtils.getTopicsURIFromConfigStore(configClient, blackListTagUri, filterString, runtimeConfig) .stream() .filter((URI u) -> ConfigUtils.getBoolean(ConfigStoreUtils.getConfig(configClient, u, runtimeConfig), KafkaSource.TOPIC_BLACKLIST, false)) .forEach(((URI u) -> blacklistedTopics.add(ConfigStoreUtils.getTopicNameFromURI(u)))); return allTopics.stream() .filter((KafkaTopic p) -> !blacklistedTopics.contains(p.getName()))
private void setTopicsFromConfigStore(State state) { Set<String> blacklistTopicsFromConfigStore = new HashSet<>(); Set<String> whitelistTopicsFromConfigStore = new HashSet<>(); ConfigStoreUtils.setTopicsFromConfigStore(state.getProperties(), blacklistTopicsFromConfigStore, whitelistTopicsFromConfigStore, MRCompactor.COMPACTION_BLACKLIST, MRCompactor.COMPACTION_WHITELIST); this.blacklist.addAll(DatasetFilterUtils.getPatternsFromStrings(new ArrayList<>(blacklistTopicsFromConfigStore))); this.whitelist.addAll(DatasetFilterUtils.getPatternsFromStrings(new ArrayList<>(whitelistTopicsFromConfigStore))); }
Optional<String> configStoreUri = getConfigStoreUri(properties); if (!configStoreUri.isPresent()) { return; Path whiteListTagUri = PathUtils.mergePaths(new Path(configStoreUri.get()), new Path(properties.getProperty(GOBBLIN_CONFIG_TAGS_WHITELIST))); getTopicsURIFromConfigStore(configClient, whiteListTagUri, filterString, runtimeConfig).stream() .filter((URI u) -> ConfigUtils.getBoolean(getConfig(configClient, u, runtimeConfig), _whitelistTopicKey, false)) .forEach(((URI u) -> whitelist.add(getTopicNameFromURI(u)))); } else if (properties.containsKey(GOBBLIN_CONFIG_TAGS_BLACKLIST)) { Preconditions.checkArgument(properties.containsKey(GOBBLIN_CONFIG_FILTER), Path blackListTagUri = PathUtils.mergePaths(new Path(configStoreUri.get()), new Path(properties.getProperty(GOBBLIN_CONFIG_TAGS_BLACKLIST))); getTopicsURIFromConfigStore(configClient, blackListTagUri, filterString, runtimeConfig).stream() .filter((URI u) -> ConfigUtils.getBoolean(getConfig(configClient, u, runtimeConfig), _blacklistTopicKey, false)) .forEach(((URI u) -> blacklist.add(getTopicNameFromURI(u)))); } else { log.warn("None of the blacklist or whitelist tags are provided");
/** * If config store is enabled, then intersection of topics from blacklisting/whitelisting will be taken against * the topics from config-store */ private List<KafkaTopic> getFilteredTopics(SourceState state) { List<Pattern> blacklist = DatasetFilterUtils.getPatternList(state, TOPIC_BLACKLIST); List<Pattern> whitelist = DatasetFilterUtils.getPatternList(state, TOPIC_WHITELIST); List<KafkaTopic> topics = this.kafkaConsumerClient.get().getFilteredTopics(blacklist, whitelist); Optional<String> configStoreUri = ConfigStoreUtils.getConfigStoreUri(state.getProperties()); if (configStoreUri.isPresent()) { List<KafkaTopic> topicsFromConfigStore = ConfigStoreUtils .getTopicsFromConfigStore(state.getProperties(), configStoreUri.get(), this.kafkaConsumerClient.get()); return topics.stream().filter((KafkaTopic p) -> (topicsFromConfigStore.stream() .anyMatch((KafkaTopic q) -> q.getName().equalsIgnoreCase(p.getName())))).collect(toList()); } return topics; }
Timer.Context context = this.metricContext.timer(CONFIG_FOR_TOPIC_TIMER).time(); configForTopic = ConfigStoreUtils.getConfigForTopic(this.props.getProperties(), KafkaSource.TOPIC_NAME, this.configClient); context.close();