public Config getConfig(String configKeyStr, Optional<Config> runtimeConfig) throws ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException, URISyntaxException { return this.getConfig(new URI(configKeyStr), runtimeConfig); }
@Override public ConfigClient call() throws Exception { return ConfigClient.createConfigClient(policy); } });
importedBys.addAll(client.getImportedBy(new URI(tag), false)); Config datasetClassConfig = client.getConfig(importedBy);
@Override public void run(String[] args) throws Exception { CliObjectFactory<Command> factory = new ConstructorAndPublicMethodsCliObjectFactory<>(Command.class); Command command = factory.buildObject(args, 1, true, args[0]); ConfigClient configClient = ConfigClient.createConfigClient(VersionStabilityPolicy.READ_FRESHEST); if (command.resolvedConfig) { Config resolvedConfig = configClient.getConfig(command.uri); System.out.println(resolvedConfig.root().render(ConfigRenderOptions.defaults())); } }
private ConfigStoreAccessor getConfigStoreAccessor(URI configKeyURI) throws ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException { URI matchedFloorKey = getMatchedFloorKeyFromCache(configKeyURI); ConfigStoreAccessor result; if (matchedFloorKey != null) { result = this.configStoreAccessorMap.get(matchedFloorKey); return result; } result = createNewConfigStoreAccessor(configKeyURI); ConfigStore cs = result.configStore; // put default root URI in cache as well for the URI which missing authority if (configKeyURI.getAuthority() == null) { // configKeyURI is missing authority/configstore root "etl-hdfs:///datasets/a1/a2" try { this.configStoreAccessorMap.put(new URI(configKeyURI.getScheme(), null, "/", null, null), result); } catch (URISyntaxException e) { // should not come here throw new RuntimeException("Can not build URI based on " + configKeyURI); } } else { // need to check Config Store's root is the prefix of input configKeyURI if (!ConfigClientUtils.isAncestorOrSame(configKeyURI, cs.getStoreURI())) { throw new RuntimeException( String.format("Config Store root URI %s is not the prefix of input %s", cs.getStoreURI(), configKeyURI)); } } // put to cache this.configStoreAccessorMap.put(cs.getStoreURI(), result); return result; }
when(mockConfigStoreFactoryRegister.getConfigStoreFactory("etl-hdfs")).thenReturn(mockConfigStoreFactory); ConfigClient client = new ConfigClient(VersionStabilityPolicy.STRONG_LOCAL_STABILITY, mockConfigStoreFactoryRegister); Config resolved = client.getConfig(relativeURI); checkValuesForIdentity(resolved); resolved = client.getConfig(absoluteURI); checkValuesForIdentity(resolved); Collection<URI> importedBy = client.getImportedBy(nertzTagURI, false); Assert.assertEquals(importedBy.size(), 1); Assert.assertEquals(importedBy.iterator().next().toString(), expectedImportedBy[0]); importedBy = client.getImportedBy(nertzTagURI, true); Assert.assertEquals(importedBy.size(), 2); for(URI u: importedBy){ "etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/data/databases/identity"}; nertzTagURI = new URI("etl-hdfs://eat1-nertznn01.grid.linkedin.com:9000/user/mitu/HdfsBasedConfigTest/tag2/nertzTag2"); importedBy = client.getImportedBy(nertzTagURI, false); Assert.assertEquals(importedBy.size(), 1); Assert.assertEquals(importedBy.iterator().next().toString(), expectedImportedBy_abs[0]); importedBy = client.getImportedBy(nertzTagURI, true); Assert.assertEquals(importedBy.size(), 2); for(URI u: importedBy){
"Missing required property " + GOBBLIN_CONFIG_TAGS_WHITELIST); String tag = this.props.getProperty(GOBBLIN_CONFIG_TAGS_WHITELIST); ConfigClient configClient = ConfigClient.createConfigClient(VersionStabilityPolicy.WEAK_LOCAL_STABILITY); Path tagUri = PathUtils.mergePaths(new Path(this.configStoreUri.get()), new Path(tag)); try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { Collection<URI> importedBy = configClient.getImportedBy(new URI(tagUri.toString()), true); for (URI uri : importedBy) { String dbName = new Path(uri).getParent().getName();
/** * Get the URIs which imports the input URI * * @param configKeyUri - The URI for the configuration key. * @param recursive - Specify whether to get direct or recursively imported by links * @return the URIs which imports the input URI * * @throws ConfigStoreFactoryDoesNotExistsException: if missing scheme name or the scheme name is invalid * @throws ConfigStoreCreationException: Specified {@link ConfigStoreFactory} can not create required {@link ConfigStore} * @throws VersionDoesNotExistException: Required version does not exist anymore ( may get deleted by retention job ) */ public Collection<URI> getImportedBy(URI configKeyUri, boolean recursive) throws ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException { return getImportedBy(configKeyUri, recursive, Optional.<Config>absent()); }
public Config getConfig(URI configKeyUri, Optional<Config> runtimeConfig) throws ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException { ConfigStoreAccessor accessor = this.getConfigStoreAccessor(configKeyUri); ConfigKeyPath configKeypath = ConfigClientUtils.buildConfigKeyPath(configKeyUri, accessor.configStore); return accessor.valueInspector.getResolvedConfig(configKeypath, runtimeConfig); }
/** * Create the {@link ConfigClient} based on the {@link VersionStabilityPolicy}. * @param policy - {@link VersionStabilityPolicy} to specify the stability policy which control the caching layer creation * @return - {@link ConfigClient} for client to use to access the {@link ConfigStore} */ public static ConfigClient createConfigClient(VersionStabilityPolicy policy) { return new ConfigClient(policy); }
@Override public void run(String[] args) throws Exception { CliObjectFactory<Command> factory = new ConstructorAndPublicMethodsCliObjectFactory<>(Command.class); Command command = factory.buildObject(args, 1, true, args[0]); ConfigClient configClient = ConfigClient.createConfigClient(VersionStabilityPolicy.READ_FRESHEST); if (command.resolvedConfig) { Config resolvedConfig = configClient.getConfig(command.uri); System.out.println(resolvedConfig.root().render(ConfigRenderOptions.defaults())); } }
"Missing required property " + GOBBLIN_CONFIG_TAGS_WHITELIST); String tag = this.props.getProperty(GOBBLIN_CONFIG_TAGS_WHITELIST); ConfigClient configClient = ConfigClient.createConfigClient(VersionStabilityPolicy.WEAK_LOCAL_STABILITY); Path tagUri = PathUtils.mergePaths(new Path(this.configStoreUri.get()), new Path(tag)); try (AutoReturnableObject<IMetaStoreClient> client = pool.getClient()) { Collection<URI> importedBy = configClient.getImportedBy(new URI(tagUri.toString()), true); for (URI uri : importedBy) { String dbName = new Path(uri).getParent().getName();
private void enhanceDisabledURIsWithBlackListTag(Set<URI> disabledURIs) throws URISyntaxException, ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException { if (this.blacklistTags.isPresent()) { for (Path s : this.blacklistTags.get()) { disabledURIs.addAll(configClient.getImportedBy(new URI(s.toString()), true)); } } }
ConfigStoreAccessor accessor = this.getConfigStoreAccessor(u); ConfigKeyPath configKeypath = ConfigClientUtils.buildConfigKeyPath(u, accessor.configStore); partitionedAccessor.put(accessor, configKeypath);
private ConfigStoreAccessor getConfigStoreAccessor(URI configKeyURI) throws ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException { URI matchedFloorKey = getMatchedFloorKeyFromCache(configKeyURI); ConfigStoreAccessor result; if (matchedFloorKey != null) { result = this.configStoreAccessorMap.get(matchedFloorKey); return result; } result = createNewConfigStoreAccessor(configKeyURI); ConfigStore cs = result.configStore; // put default root URI in cache as well for the URI which missing authority if (configKeyURI.getAuthority() == null) { // configKeyURI is missing authority/configstore root "etl-hdfs:///datasets/a1/a2" try { this.configStoreAccessorMap.put(new URI(configKeyURI.getScheme(), null, "/", null, null), result); } catch (URISyntaxException e) { // should not come here throw new RuntimeException("Can not build URI based on " + configKeyURI); } } else { // need to check Config Store's root is the prefix of input configKeyURI if (!ConfigClientUtils.isAncestorOrSame(configKeyURI, cs.getStoreURI())) { throw new RuntimeException( String.format("Config Store root URI %s is not the prefix of input %s", cs.getStoreURI(), configKeyURI)); } } // put to cache this.configStoreAccessorMap.put(cs.getStoreURI(), result); return result; }
/** * Create the {@link ConfigClient} based on the {@link VersionStabilityPolicy}. * @param policy - {@link VersionStabilityPolicy} to specify the stability policy which control the caching layer creation * @return - {@link ConfigClient} for client to use to access the {@link ConfigStore} */ public static ConfigClient createConfigClient(VersionStabilityPolicy policy) { return new ConfigClient(policy); }
/** * Convenient method to get resolved {@link Config} based on String input. */ public Config getConfig(String configKeyStr) throws ConfigStoreFactoryDoesNotExistsException, ConfigStoreCreationException, VersionDoesNotExistException, URISyntaxException { return getConfig(configKeyStr, Optional.<Config>absent()); }
configClient = ConfigClient.createConfigClient(VersionStabilityPolicy.WEAK_LOCAL_STABILITY); this.props = jobProps;
importedBys.addAll(client.getImportedBy(new URI(tag), false)); Config datasetClassConfig = client.getConfig(importedBy);
/** * Will return the list of URIs given which are importing tag {@param tagUri} */ public static Collection<URI> getTopicsURIFromConfigStore(ConfigClient configClient, Path tagUri, String filterString, Optional<Config> runtimeConfig) { try { Collection<URI> importedBy = configClient.getImportedBy(new URI(tagUri.toString()), true, runtimeConfig); return importedBy.stream().filter((URI u) -> u.toString().contains(filterString)).collect(Collectors.toList()); } catch (URISyntaxException | ConfigStoreFactoryDoesNotExistsException | ConfigStoreCreationException e) { throw new Error(e); } }