private String extractAppVersionProperty(StreamAppDefinition appDefinition, Map<String, String> updateProperties) { String versionPrefix = String.format("version.%s", appDefinition.getName()); for (Map.Entry<String, String> entry : updateProperties.entrySet()) { if (entry.getKey().startsWith(versionPrefix)) { return entry.getValue(); } } return null; }
private String extractAppVersionProperty(StreamAppDefinition appDefinition, Map<String, String> updateProperties) { String versionPrefix = String.format("version.%s", appDefinition.getName()); for (Map.Entry<String, String> entry : updateProperties.entrySet()) { if (entry.getKey().startsWith(versionPrefix)) { return entry.getValue(); } } return null; }
/** * Determines a deployment key for a stream application. * * @param streamAppDefinition the stream application definition * @return the deployment key */ public static String forStreamAppDefinition(StreamAppDefinition streamAppDefinition) { Assert.notNull(streamAppDefinition, "streamAppDefinition must not be null"); return String.format("%s.%s", streamAppDefinition.getStreamName(), streamAppDefinition.getName()); }
private String getRegisteredName(StreamDefinition streamDefinition, String adrAppName) { for (StreamAppDefinition appDefinition: streamDefinition.getAppDefinitions()) { if (appDefinition.getName().equals(adrAppName)) { return appDefinition.getRegisteredAppName(); } } return adrAppName; }
private String getRegisteredName(StreamDefinition streamDefinition, String adrAppName) { for (StreamAppDefinition appDefinition : streamDefinition.getAppDefinitions()) { if (appDefinition.getName().equals(adrAppName)) { return appDefinition.getRegisteredAppName(); } } return adrAppName; }
/** * Extract and return a map of properties for a specific app within the deployment * properties of a stream. * * @param appDefinition the {@link StreamAppDefinition} for which to return a map of * properties * @param streamDeploymentProperties deployment properties for the stream that the app is * defined in * @return map of properties for an app */ /* default */ Map<String, String> extractAppProperties(StreamAppDefinition appDefinition, Map<String, String> streamDeploymentProperties) { Map<String, String> appDeploymentProperties = new HashMap<>(); appDeploymentProperties.putAll(this.commonApplicationProperties.getStream()); // add properties with wild card prefix String wildCardProducerPropertyPrefix = "app.*.producer."; String wildCardConsumerPropertyPrefix = "app.*.consumer."; String wildCardPrefix = "app.*."; parseAndPopulateProperties(streamDeploymentProperties, appDeploymentProperties, wildCardProducerPropertyPrefix, wildCardConsumerPropertyPrefix, wildCardPrefix); // add application specific properties String producerPropertyPrefix = String.format("app.%s.producer.", appDefinition.getName()); String consumerPropertyPrefix = String.format("app.%s.consumer.", appDefinition.getName()); String appPrefix = String.format("app.%s.", appDefinition.getName()); parseAndPopulateProperties(streamDeploymentProperties, appDeploymentProperties, producerPropertyPrefix, consumerPropertyPrefix, appPrefix); return appDeploymentProperties; }
/** * Return a new app definition where definition-time and deploy-time properties have been * merged and short form parameters have been expanded to their long form (amongst the * whitelisted supported properties of the app) if applicable. */ /* default */ AppDefinition mergeAndExpandAppProperties(StreamAppDefinition original, Resource metadataResource, Map<String, String> appDeployTimeProperties) { Map<String, String> merged = new HashMap<>(original.getProperties()); merged.putAll(appDeployTimeProperties); merged = this.whitelistProperties.qualifyProperties(merged, metadataResource); merged.putIfAbsent(StreamPropertyKeys.METRICS_PROPERTIES, "spring.application.name,spring.application.index," + "spring.cloud.application.*,spring.cloud.dataflow.*"); merged.putIfAbsent(StreamPropertyKeys.METRICS_TRIGGER_INCLUDES, "integration**"); return new AppDefinition(original.getName(), merged); }
/** * Return a new app definition where definition-time and deploy-time properties have been * merged and short form parameters have been expanded to their long form (amongst the * whitelisted supported properties of the app) if applicable. */ /* default */ AppDefinition mergeAndExpandAppProperties(StreamAppDefinition original, Resource metadataResource, Map<String, String> appDeployTimeProperties) { Map<String, String> merged = new HashMap<>(original.getProperties()); merged.putAll(appDeployTimeProperties); merged = this.whitelistProperties.qualifyProperties(merged, metadataResource); merged.putIfAbsent(StreamPropertyKeys.METRICS_PROPERTIES, "spring.application.name,spring.application.index," + "spring.cloud.application.*,spring.cloud.dataflow.*"); merged.putIfAbsent(StreamPropertyKeys.METRICS_TRIGGER_INCLUDES, "integration**"); return new AppDefinition(original.getName(), merged); }
@Override public boolean addProposals(String text, StreamDefinition streamDefinition, int detailLevel, List<CompletionProposal> collector) { StreamAppDefinition lastApp = streamDefinition.getDeploymentOrderIterator().next(); AppRegistration appRegistration = this.collectorSupport.findAppRegistration(lastApp.getName(), CompletionUtils.determinePotentialTypes(lastApp,streamDefinition.getAppDefinitions().size() > 1)); if (appRegistration != null) { Set<String> alreadyPresentOptions = new HashSet<>(lastApp.getProperties().keySet()); this.collectorSupport.addPropertiesProposals(text, "", appRegistration, alreadyPresentOptions, collector, detailLevel); } return false; } }
@Override public boolean addProposals(String text, StreamDefinition parseResult, int detailLevel, List<CompletionProposal> collector) { Set<String> propertyNames = new HashSet<>( parseResult.getDeploymentOrderIterator().next().getProperties().keySet()); propertyNames.removeAll(CompletionUtils.IMPLICIT_PARAMETER_NAMES); if (text.endsWith(" ") || propertyNames.isEmpty()) { return false; } String propertyName = recoverPropertyName(text); StreamAppDefinition lastApp = parseResult.getDeploymentOrderIterator().next(); String alreadyTyped = lastApp.getProperties().get(propertyName); AppRegistration lastAppRegistration = this.collectorSupport.findAppRegistration(lastApp.getName(), CompletionUtils.determinePotentialTypes(lastApp, parseResult.getAppDefinitions().size() > 1)); if (lastAppRegistration != null) { return this.collectorSupport.addAlreadyTypedValueHintsProposals(text, lastAppRegistration, collector, propertyName, valueHintProviders, alreadyTyped); } return false; }
Set<String> parameterNames = new HashSet<>(lastApp.getProperties().keySet()); parameterNames.removeAll(CompletionUtils.IMPLICIT_PARAMETER_NAMES); if (!parameterNames.isEmpty() || !text.endsWith(lastApp.getName())) { return false; String alreadyTyped = lastApp.getName(); CompletionProposal.Factory proposals = CompletionProposal.expanding(text);
/** * Return the {@link ApplicationType} for a {@link AppDefinition} in the context of a * defined stream. * * @param appDefinition the app for which to determine the type * @return {@link ApplicationType} for the given app * @throws CannotDetermineApplicationTypeException Thrown if the * {@link ApplicationType} cannot be determined */ public static ApplicationType determineApplicationType(StreamAppDefinition appDefinition) { // Parser has already taken care of source/sink destinations, etc boolean hasOutput = appDefinition.getProperties().containsKey(BindingPropertyKeys.OUTPUT_DESTINATION); boolean hasInput = appDefinition.getProperties().containsKey(BindingPropertyKeys.INPUT_DESTINATION); if (hasInput && hasOutput) { return ApplicationType.processor; } else if (hasInput) { return ApplicationType.sink; } else if (hasOutput) { return ApplicationType.source; } else { throw new CannotDetermineApplicationTypeException( appDefinition.getName() + " had neither input nor " + "output set"); } } }
private AppRegistration lookupLastApp(CheckPointedParseException exception) { String safe = exception.getExpressionStringUntilCheckpoint(); StreamDefinition streamDefinition = new StreamDefinition("__dummy", safe); StreamAppDefinition lastApp = streamDefinition.getDeploymentOrderIterator().next(); return this.collectorSupport.findAppRegistration(lastApp.getName(), CompletionUtils.determinePotentialTypes(lastApp, streamDefinition.getAppDefinitions().size() > 1)); }
@Override public void addProposals(String dsl, CheckPointedParseException exception, int detailLevel, List<CompletionProposal> collector) { String safe = exception.getExpressionStringUntilCheckpoint(); StreamDefinition streamDefinition = new StreamDefinition("__dummy", safe); StreamAppDefinition lastApp = streamDefinition.getDeploymentOrderIterator().next(); AppRegistration appRegistration = this.collectorSupport.findAppRegistration(lastApp.getName(), CompletionUtils.determinePotentialTypes(lastApp, streamDefinition.getAppDefinitions().size() > 1)); if (appRegistration != null) { Set<String> alreadyPresentOptions = new HashSet<>(lastApp.getProperties().keySet()); this.collectorSupport.addPropertiesProposals(safe, "", appRegistration, alreadyPresentOptions, collector, detailLevel); } } }
@Override public void addProposals(String dsl, CheckPointedParseException exception, int detailLevel, List<CompletionProposal> collector) { String safe = exception.getExpressionStringUntilCheckpoint(); StreamDefinition streamDefinition = new StreamDefinition("__dummy", safe); StreamAppDefinition lastApp = streamDefinition.getDeploymentOrderIterator().next(); AppRegistration appRegistration = this.collectorSupport.findAppRegistration(lastApp.getName(), CompletionUtils.determinePotentialTypes(lastApp, streamDefinition.getAppDefinitions().size() > 1)); if (appRegistration != null) { String startsWith = ProposalsCollectorSupportUtils.computeStartsWith(exception); Set<String> alreadyPresentOptions = new HashSet<>(lastApp.getProperties().keySet()); this.collectorSupport.addPropertiesProposals(safe, startsWith, appRegistration, alreadyPresentOptions, collector, detailLevel); } } }
@Override public ValidationStatus validateStream(String name) { StreamDefinition definition = streamDefinitionRepository.findById(name) .orElseThrow(() -> new NoSuchStreamDefinitionException(name)); ValidationStatus validationStatus = new ValidationStatus( definition.getName(), definition.getDslText()); for (StreamAppDefinition streamAppDefinition : definition.getAppDefinitions()) { ApplicationType appType = streamAppDefinition.getApplicationType(); boolean status = this.validate(streamAppDefinition.getName(), appType); validationStatus.getAppsStatuses().put( String.format("%s:%s", appType.name(), streamAppDefinition.getName()), (status) ? NodeStatus.valid.name() : NodeStatus.invalid.name()); } return validationStatus; } }
@Override public boolean addProposals(String text, StreamDefinition parseResult, int detailLevel, List<CompletionProposal> collector) { if (text.isEmpty() || !text.endsWith(" ")) { return false; } StreamAppDefinition lastApp = parseResult.getDeploymentOrderIterator().next(); // Consider "bar | foo". If there is indeed a sink named foo in the registry, // "foo" may also be a processor, in which case we can continue boolean couldBeASink = appRegistry.find(lastApp.getName(), ApplicationType.sink) != null; if (couldBeASink) { boolean couldBeAProcessor = appRegistry.find(lastApp.getName(), ApplicationType.processor) != null; if (!couldBeAProcessor) { return false; } } CompletionProposal.Factory proposals = CompletionProposal.expanding(text); for (AppRegistration appRegistration : appRegistry.findAll()) { if (appRegistration.getType() == ApplicationType.processor || appRegistration.getType() == ApplicationType.sink) { String expansion = CompletionUtils.maybeQualifyWithLabel(appRegistration.getName(), parseResult); collector.add(proposals.withSeparateTokens("| " + expansion, "Continue stream definition with a " + appRegistration.getType())); } } return false; } }
@Override public ValidationStatus validateStream(String name) { StreamDefinition definition = streamDefinitionRepository.findOne(name); if (definition == null) { throw new NoSuchStreamDefinitionException(name); } ValidationStatus validationStatus = new ValidationStatus( definition.getName(), definition.getDslText()); for (StreamAppDefinition streamAppDefinition : definition.getAppDefinitions()) { ApplicationType appType = streamAppDefinition.getApplicationType(); boolean status = this.validate(streamAppDefinition.getName(), appType); validationStatus.getAppsStatuses().put( String.format("%s:%s", appType.name(), streamAppDefinition.getName()), (status) ? NodeStatus.valid.name() : NodeStatus.invalid.name()); } return validationStatus; } }
private void updateStreamDefinitionFromReleaseManifest(String streamName, String releaseManifest) { List<SpringCloudDeployerApplicationManifest> appManifests = new SpringCloudDeployerApplicationManifestReader() .read(releaseManifest); Map<String, SpringCloudDeployerApplicationManifest> appManifestMap = new HashMap<>(); for (SpringCloudDeployerApplicationManifest am : appManifests) { String name = am.getSpec().getApplicationProperties().get(DataFlowPropertyKeys.STREAM_APP_LABEL); appManifestMap.put(name, am); } StreamDefinition streamDefinition = this.streamDefinitionRepository.findById(streamName) .orElseThrow(() -> new NoSuchStreamDefinitionException(streamName)); LinkedList<StreamAppDefinition> updatedStreamAppDefinitions = new LinkedList<>(); for (StreamAppDefinition appDefinition : streamDefinition.getAppDefinitions()) { StreamAppDefinition.Builder appDefinitionBuilder = StreamAppDefinition.Builder.from(appDefinition); SpringCloudDeployerApplicationManifest applicationManifest = appManifestMap.get(appDefinition.getName()); // overrides app definition properties with those from the release manifest appDefinitionBuilder.setProperties(applicationManifest.getSpec().getApplicationProperties()); updatedStreamAppDefinitions.addLast(appDefinitionBuilder.build(streamDefinition.getName())); } String dslText = new StreamDefinitionToDslConverter().toDsl(updatedStreamAppDefinitions); StreamDefinition updatedStreamDefinition = new StreamDefinition(streamName, dslText); logger.debug("Updated StreamDefinition: " + updatedStreamDefinition); // TODO consider adding an explicit UPDATE method to the streamDefRepository // Note: Not transactional and can lead to loosing the stream definition this.streamDefinitionRepository.delete(updatedStreamDefinition); this.streamDefinitionRepository.save(updatedStreamDefinition); this.auditRecordService.populateAndSaveAuditRecord( AuditOperationType.STREAM, AuditActionType.UPDATE, streamName, this.auditServiceUtils.convertStreamDefinitionToAuditData(streamDefinition)); }
/** * Create a new builder that is initialized with properties of the given * definition. Useful for "mutating" a definition by building a slightly different * copy. * * @param definition the StreamAppDefinition to create a new Builder instance with * stream name, registeredAppName, label, and properties populated * @return a StreamAppDefinition builder */ public static Builder from(StreamAppDefinition definition) { Builder builder = new Builder(); builder.setStreamName(definition.getStreamName()).setRegisteredAppName(definition.getRegisteredAppName()) .setApplicationType(definition.getApplicationType()) .setLabel(definition.getName()).addProperties(definition.getProperties()); return builder; }