private Config getResolvedConfigHelper(Config userConfig, Set<JobTemplate> alreadyLoadedTemplates) throws SpecNotFoundException, TemplateException { Config config = getLocallyResolvedConfig(userConfig); for (JobTemplate template : Lists.reverse(this.superTemplates)) { if (!alreadyLoadedTemplates.contains(template)) { alreadyLoadedTemplates.add(template); Config fallback = template instanceof InheritingJobTemplate ? ((InheritingJobTemplate) template).getResolvedConfigHelper(config, alreadyLoadedTemplates) : template.getResolvedConfig(config); config = config.withFallback(fallback); } } return config; }
@Override public List<Config> getResolvedJobConfigs(Config userConfig, DatasetDescriptor inputDescriptor, DatasetDescriptor outputDescriptor) throws SpecNotFoundException, JobTemplate.TemplateException { Config inputDescriptorConfig = inputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_INPUT_DATASET_DESCRIPTOR_PREFIX); Config outputDescriptorConfig = outputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_OUTPUT_DATASET_DESCRIPTOR_PREFIX); userConfig = userConfig.withFallback(inputDescriptorConfig).withFallback(outputDescriptorConfig); List<Config> resolvedJobConfigs = new ArrayList<>(); for (JobTemplate jobTemplate: getJobTemplates()) { Config resolvedJobConfig = jobTemplate.getResolvedConfig(userConfig).resolve().withValue( ConfigurationKeys.JOB_TEMPLATE_PATH, ConfigValueFactory.fromAnyRef(jobTemplate.getUri().toString()));; resolvedJobConfigs.add(resolvedJobConfig); } return resolvedJobConfigs; } }
private static Config resolveConfig(JobSpec jobSpec, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException { Optional<URI> templateURIOpt = jobSpec.getTemplateURI(); if (templateURIOpt.isPresent()) { JobCatalogWithTemplates catalogWithTemplates = new PackagedTemplatesJobCatalogDecorator(catalog); JobTemplate template = catalogWithTemplates.getTemplate(templateURIOpt.get()); return template.getResolvedConfig(jobSpec.getConfig()).resolve(); } else { return jobSpec.getConfig().resolve(); } }
URI templateUri = new URI(jobProps.getProperty(TEMPLATE_KEY)); Config resolvedJob = new PackagedTemplatesJobCatalogDecorator().getTemplate(templateUri) .getResolvedConfig(ConfigUtils.propertiesToConfig(jobProps)); jobProps = ConfigUtils.configToProperties(resolvedJob);
/** * Checks if the {@link FlowTemplate} is resolvable using the provided {@link Config} object. A {@link FlowTemplate} * is resolvable only if each of the {@link JobTemplate}s in the flow is resolvable * @param userConfig User supplied Config * @return true if the {@link FlowTemplate} is resolvable */ @Override public boolean isResolvable(Config userConfig, DatasetDescriptor inputDescriptor, DatasetDescriptor outputDescriptor) throws SpecNotFoundException, JobTemplate.TemplateException { Config inputDescriptorConfig = inputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_INPUT_DATASET_DESCRIPTOR_PREFIX); Config outputDescriptorConfig = outputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_OUTPUT_DATASET_DESCRIPTOR_PREFIX); userConfig = userConfig.withFallback(inputDescriptorConfig).withFallback(outputDescriptorConfig); ConfigResolveOptions resolveOptions = ConfigResolveOptions.defaults().setAllowUnresolved(true); for (JobTemplate template: this.jobTemplates) { Config templateConfig = template.getResolvedConfig(userConfig).resolve(resolveOptions); if (!template.getResolvedConfig(userConfig).resolve(resolveOptions).isResolved()) { return false; } } return true; }
if (this.template != null) { try { finalConfig = this.template.getResolvedConfig(finalConfig); } catch (SpecNotFoundException | JobTemplate.TemplateException exc) { throw new RuntimeException(exc);
private Config getResolvedConfigHelper(Config userConfig, Set<JobTemplate> alreadyLoadedTemplates) throws SpecNotFoundException, TemplateException { Config config = getLocallyResolvedConfig(userConfig); for (JobTemplate template : Lists.reverse(this.superTemplates)) { if (!alreadyLoadedTemplates.contains(template)) { alreadyLoadedTemplates.add(template); Config fallback = template instanceof InheritingJobTemplate ? ((InheritingJobTemplate) template).getResolvedConfigHelper(config, alreadyLoadedTemplates) : template.getResolvedConfig(config); config = config.withFallback(fallback); } } return config; }
private static Config resolveConfig(JobSpec jobSpec, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException { Optional<URI> templateURIOpt = jobSpec.getTemplateURI(); if (templateURIOpt.isPresent()) { JobCatalogWithTemplates catalogWithTemplates = new PackagedTemplatesJobCatalogDecorator(catalog); JobTemplate template = catalogWithTemplates.getTemplate(templateURIOpt.get()); return template.getResolvedConfig(jobSpec.getConfig()).resolve(); } else { return jobSpec.getConfig().resolve(); } }
@Override public List<Config> getResolvedJobConfigs(Config userConfig, DatasetDescriptor inputDescriptor, DatasetDescriptor outputDescriptor) throws SpecNotFoundException, JobTemplate.TemplateException { Config inputDescriptorConfig = inputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_INPUT_DATASET_DESCRIPTOR_PREFIX); Config outputDescriptorConfig = outputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_OUTPUT_DATASET_DESCRIPTOR_PREFIX); userConfig = userConfig.withFallback(inputDescriptorConfig).withFallback(outputDescriptorConfig); List<Config> resolvedJobConfigs = new ArrayList<>(); for (JobTemplate jobTemplate: getJobTemplates()) { Config resolvedJobConfig = jobTemplate.getResolvedConfig(userConfig).resolve().withValue( ConfigurationKeys.JOB_TEMPLATE_PATH, ConfigValueFactory.fromAnyRef(jobTemplate.getUri().toString()));; resolvedJobConfigs.add(resolvedJobConfig); } return resolvedJobConfigs; } }
URI templateUri = new URI(jobProps.getProperty(TEMPLATE_KEY)); Config resolvedJob = new PackagedTemplatesJobCatalogDecorator().getTemplate(templateUri) .getResolvedConfig(ConfigUtils.propertiesToConfig(jobProps)); jobProps = ConfigUtils.configToProperties(resolvedJob);
/** * Checks if the {@link FlowTemplate} is resolvable using the provided {@link Config} object. A {@link FlowTemplate} * is resolvable only if each of the {@link JobTemplate}s in the flow is resolvable * @param userConfig User supplied Config * @return true if the {@link FlowTemplate} is resolvable */ @Override public boolean isResolvable(Config userConfig, DatasetDescriptor inputDescriptor, DatasetDescriptor outputDescriptor) throws SpecNotFoundException, JobTemplate.TemplateException { Config inputDescriptorConfig = inputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_INPUT_DATASET_DESCRIPTOR_PREFIX); Config outputDescriptorConfig = outputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_OUTPUT_DATASET_DESCRIPTOR_PREFIX); userConfig = userConfig.withFallback(inputDescriptorConfig).withFallback(outputDescriptorConfig); ConfigResolveOptions resolveOptions = ConfigResolveOptions.defaults().setAllowUnresolved(true); for (JobTemplate template: this.jobTemplates) { Config templateConfig = template.getResolvedConfig(userConfig).resolve(resolveOptions); if (!template.getResolvedConfig(userConfig).resolve(resolveOptions).isResolved()) { return false; } } return true; }
if (this.template != null) { try { finalConfig = this.template.getResolvedConfig(finalConfig); } catch (SpecNotFoundException | JobTemplate.TemplateException exc) { throw new RuntimeException(exc);