private AppRegistration lookupLastApp(CheckPointedParseException exception) { String safe = exception.getExpressionStringUntilCheckpoint(); StreamDefinition streamDefinition = new StreamDefinition("__dummy", safe); StreamAppDefinition lastApp = streamDefinition.getDeploymentOrderIterator().next(); return this.collectorSupport.findAppRegistration(lastApp.getName(), CompletionUtils.determinePotentialTypes(lastApp, streamDefinition.getAppDefinitions().size() > 1)); }
@Override public <S extends StreamDefinition> S save(S definition) { definitions.put(definition.getName(), definition); return definition; }
@Override public <S extends StreamDefinition> S save(S entity) { hashOperations.put(entity.getName(), entity.getDslText()); return entity; }
@Override public boolean addProposals(String text, StreamDefinition streamDefinition, int detailLevel, List<CompletionProposal> collector) { StreamAppDefinition lastApp = streamDefinition.getDeploymentOrderIterator().next(); AppRegistration appRegistration = this.collectorSupport.findAppRegistration(lastApp.getName(), CompletionUtils.determinePotentialTypes(lastApp,streamDefinition.getAppDefinitions().size() > 1)); if (appRegistration != null) { Set<String> alreadyPresentOptions = new HashSet<>(lastApp.getProperties().keySet()); this.collectorSupport.addPropertiesProposals(text, "", appRegistration, alreadyPresentOptions, collector, detailLevel); } return false; } }
/** * Redacts sensitive property values in a stream. * * @param streamDefinition the stream definition to sanitize * @return Stream definition text that has sensitive data redacted. */ public String sanitizeStream(StreamDefinition streamDefinition) { List<StreamAppDefinition> sanitizedAppDefinitions = streamDefinition.getAppDefinitions().stream() .map(app -> StreamAppDefinition.Builder .from(app) .setProperties(this.sanitizeProperties(app.getProperties())) .build(streamDefinition.getName()) ).collect(Collectors.toList()); return this.streamDslConverter.toDsl(sanitizedAppDefinitions); }
private void updateStreamDefinitionFromReleaseManifest(String streamName, String releaseManifest) { List<SpringCloudDeployerApplicationManifest> appManifests = new SpringCloudDeployerApplicationManifestReader() .read(releaseManifest); Map<String, SpringCloudDeployerApplicationManifest> appManifestMap = new HashMap<>(); for (SpringCloudDeployerApplicationManifest am : appManifests) { String name = am.getSpec().getApplicationProperties().get(DataFlowPropertyKeys.STREAM_APP_LABEL); appManifestMap.put(name, am); } StreamDefinition streamDefinition = this.streamDefinitionRepository.findById(streamName) .orElseThrow(() -> new NoSuchStreamDefinitionException(streamName)); LinkedList<StreamAppDefinition> updatedStreamAppDefinitions = new LinkedList<>(); for (StreamAppDefinition appDefinition : streamDefinition.getAppDefinitions()) { StreamAppDefinition.Builder appDefinitionBuilder = StreamAppDefinition.Builder.from(appDefinition); SpringCloudDeployerApplicationManifest applicationManifest = appManifestMap.get(appDefinition.getName()); // overrides app definition properties with those from the release manifest appDefinitionBuilder.setProperties(applicationManifest.getSpec().getApplicationProperties()); updatedStreamAppDefinitions.addLast(appDefinitionBuilder.build(streamDefinition.getName())); } String dslText = new StreamDefinitionToDslConverter().toDsl(updatedStreamAppDefinitions); StreamDefinition updatedStreamDefinition = new StreamDefinition(streamName, dslText); logger.debug("Updated StreamDefinition: " + updatedStreamDefinition); // TODO consider adding an explicit UPDATE method to the streamDefRepository // Note: Not transactional and can lead to loosing the stream definition this.streamDefinitionRepository.delete(updatedStreamDefinition); this.streamDefinitionRepository.save(updatedStreamDefinition); this.auditRecordService.populateAndSaveAuditRecord( AuditOperationType.STREAM, AuditActionType.UPDATE, streamName, this.auditServiceUtils.convertStreamDefinitionToAuditData(streamDefinition)); }
@Override public ValidationStatus validateStream(String name) { StreamDefinition definition = streamDefinitionRepository.findById(name) .orElseThrow(() -> new NoSuchStreamDefinitionException(name)); ValidationStatus validationStatus = new ValidationStatus( definition.getName(), definition.getDslText()); for (StreamAppDefinition streamAppDefinition : definition.getAppDefinitions()) { ApplicationType appType = streamAppDefinition.getApplicationType(); boolean status = this.validate(streamAppDefinition.getName(), appType); validationStatus.getAppsStatuses().put( String.format("%s:%s", appType.name(), streamAppDefinition.getName()), (status) ? NodeStatus.valid.name() : NodeStatus.invalid.name()); } return validationStatus; } }
/** * Return the ordered list of application definitions for this stream as a * {@link List}. This allows for retrieval of application definitions in the stream by * index. Application definitions are maintained in stream flow order (source is * first, sink is last). * * @return list of application definitions for this stream definition */ public LinkedList<StreamAppDefinition> getAppDefinitions() { if (CollectionUtils.isEmpty(this.applicationDefinitions)) { return getAppDefinitions(this.name, this.dslText); } return this.applicationDefinitions; }
@Override public StreamDefinition mapRow(ResultSet resultSet, int i) throws SQLException { return new StreamDefinition(resultSet.getString("DEFINITION_NAME"), resultSet.getString("DEFINITION")); } }, "DEFINITION_NAME", "DEFINITION");
/** * Return {@code true} if the upstream module (the module that appears before * the provided module) contains partition related properties. * * @param stream stream for the module * @param currentModule module for which to determine if the upstream module * has partition properties * @param streamDeploymentProperties deployment properties for the stream * @return true if the upstream module has partition properties */ private boolean upstreamModuleHasPartitionInfo(StreamDefinition stream, ModuleDefinition currentModule, Map<String, String> streamDeploymentProperties) { Iterator<ModuleDefinition> iterator = stream.getDeploymentOrderIterator(); while (iterator.hasNext()) { ModuleDefinition module = iterator.next(); if (module.equals(currentModule) && iterator.hasNext()) { ModuleDefinition prevModule = iterator.next(); Map<String, String> moduleDeploymentProperties = extractModuleDeploymentProperties(prevModule, streamDeploymentProperties); return moduleDeploymentProperties.containsKey(PARTITION_KEY_EXPRESSION) || moduleDeploymentProperties.containsKey(PARTITION_KEY_EXTRACTOR_CLASS); } } return false; }
for (ModuleDefinition moduleDefinition : streamDefinition.getModuleDefinitions()) { if (moduleDefinition.getLabel().startsWith(moduleName)) { collector.add(proposals.withSuffix(moduleDefinition.getLabel())); CompletionProposal.Factory proposals = CompletionProposal.expanding("tap:stream:"); for (StreamDefinition stream : streamDefinitionRepository.findAll()) { if (stream.getName().startsWith(streamName)) { collector.add(proposals.withSuffix(stream.getName()));
/** * Request deployment of an existing stream definition. * @param name the name of an existing stream definition (required) * @return The stream deployment */ @RequestMapping(value = "/{name}", method = RequestMethod.GET) @ResponseStatus(HttpStatus.CREATED) public StreamDeploymentResource info(@PathVariable("name") String name) { StreamDefinition streamDefinition = this.repository.findById(name) .orElseThrow(() -> new NoSuchStreamDefinitionException(name)); StreamDeployment streamDeployment = this.streamService.info(name); Map<StreamDefinition, DeploymentState> streamDeploymentStates = this.streamService.state(Arrays.asList(streamDefinition)); DeploymentState deploymentState = streamDeploymentStates.get(streamDefinition); String status = ""; if (deploymentState != null) { final DeploymentStateResource deploymentStateResource = ControllerUtils.mapState(deploymentState); status = deploymentStateResource.getKey(); } return new Assembler(streamDefinition.getDslText(), status).toResource(streamDeployment); }
/** * Undeploy the given stream. * * @param stream stream to undeploy */ private void undeployStream(StreamDefinition stream) { for (ModuleDefinition module : stream.getModuleDefinitions()) { ModuleDeploymentId id = ModuleDeploymentId.fromModuleDefinition(module); ModuleStatus status = this.deployer.status(id); if (!EnumSet.of(DeploymentState.unknown, DeploymentState.undeployed) .contains(status.getState())) { this.deployer.undeploy(id); } } }
StreamDeployment streamDeployment = this.streamService.info(streamDefinition.getName()); for (StreamAppDefinition streamAppDefinition : streamDefinition.getAppDefinitions()) { final String streamAppName = streamAppDefinition.getRegisteredAppName(); final ApplicationType streamAppType = streamAppDefinition.getApplicationType(); String version = appDeploymentProperties.get(SkipperStream.SKIPPER_SPEC_VERSION); if (version != null && version.equals(appVersion)) { return streamDefinition.getName();
private void updateStreamDefinitionFromReleaseManifest(String streamName, String releaseManifest) { List<SpringCloudDeployerApplicationManifest> appManifests = new SpringCloudDeployerApplicationManifestReader() .read(releaseManifest); Map<String, SpringCloudDeployerApplicationManifest> appManifestMap = new HashMap<>(); for (SpringCloudDeployerApplicationManifest am : appManifests) { String name = am.getSpec().getApplicationProperties().get(DataFlowPropertyKeys.STREAM_APP_LABEL); appManifestMap.put(name, am); } StreamDefinition streamDefinition = this.streamDefinitionRepository.findOne(streamName); LinkedList<StreamAppDefinition> updatedStreamAppDefinitions = new LinkedList<>(); for (StreamAppDefinition appDefinition : streamDefinition.getAppDefinitions()) { StreamAppDefinition.Builder appDefinitionBuilder = StreamAppDefinition.Builder.from(appDefinition); SpringCloudDeployerApplicationManifest applicationManifest = appManifestMap.get(appDefinition.getName()); // overrides app definition properties with those from the release manifest appDefinitionBuilder.setProperties(applicationManifest.getSpec().getApplicationProperties()); updatedStreamAppDefinitions.addLast(appDefinitionBuilder.build(streamDefinition.getName())); } String dslText = new StreamDefinitionToDslConverter().toDsl(updatedStreamAppDefinitions); StreamDefinition updatedStreamDefinition = new StreamDefinition(streamName, dslText); logger.debug("Updated StreamDefinition: " + updatedStreamDefinition); // TODO consider adding an explicit UPDATE method to the streamDefRepository // Note: Not transactional and can lead to loosing the stream definition this.streamDefinitionRepository.delete(updatedStreamDefinition); this.streamDefinitionRepository.save(updatedStreamDefinition); this.auditRecordService.populateAndSaveAuditRecord( AuditOperationType.STREAM, AuditActionType.UPDATE, streamName, this.auditServiceUtils.convertStreamDefinitionToAuditData(streamDefinition)); }
@Override public ValidationStatus validateStream(String name) { StreamDefinition definition = streamDefinitionRepository.findOne(name); if (definition == null) { throw new NoSuchStreamDefinitionException(name); } ValidationStatus validationStatus = new ValidationStatus( definition.getName(), definition.getDslText()); for (StreamAppDefinition streamAppDefinition : definition.getAppDefinitions()) { ApplicationType appType = streamAppDefinition.getApplicationType(); boolean status = this.validate(streamAppDefinition.getName(), appType); validationStatus.getAppsStatuses().put( String.format("%s:%s", appType.name(), streamAppDefinition.getName()), (status) ? NodeStatus.valid.name() : NodeStatus.invalid.name()); } return validationStatus; } }
/** * Construct a {@code StreamDefinition}. * * @param name name of stream * @param dslText DSL definition for stream */ public StreamDefinition(String name, String dslText) { Assert.hasText(name, "name is required"); Assert.hasText(dslText, "dslText is required"); this.name = name; this.dslText = dslText; this.applicationDefinitions = getAppDefinitions(name, dslText); }
@Override public boolean addProposals(String text, StreamDefinition parseResult, int detailLevel, List<CompletionProposal> collector) { Set<String> propertyNames = new HashSet<>( parseResult.getDeploymentOrderIterator().next().getProperties().keySet()); propertyNames.removeAll(CompletionUtils.IMPLICIT_PARAMETER_NAMES); if (text.endsWith(" ") || propertyNames.isEmpty()) { return false; } String propertyName = recoverPropertyName(text); StreamAppDefinition lastApp = parseResult.getDeploymentOrderIterator().next(); String alreadyTyped = lastApp.getProperties().get(propertyName); AppRegistration lastAppRegistration = this.collectorSupport.findAppRegistration(lastApp.getName(), CompletionUtils.determinePotentialTypes(lastApp, parseResult.getAppDefinitions().size() > 1)); if (lastAppRegistration != null) { return this.collectorSupport.addAlreadyTypedValueHintsProposals(text, lastAppRegistration, collector, propertyName, valueHintProviders, alreadyTyped); } return false; }
/** * Return a list of {@link StreamDefinition}s, made by mapping a set of non * null names and possibly null dsl texts. In case of null dsl text, * a null element is added to the result list. * * @param names list of stream names * @param definitions list of stream definitions */ private List<StreamDefinition> zipToStreamDefinitions(List<String> names, List<String> definitions) { Iterator<String> it = definitions.iterator(); List<StreamDefinition> result = new ArrayList<>(definitions.size()); for (String name : names) { String dsl = it.next(); result.add(dsl != null ? new StreamDefinition(name, dsl) : null); } return result; }
/** * Return {@code true} if the upstream app (the app that appears before the provided app) * contains partition related properties. * * @param stream stream for the app * @param currentApp app for which to determine if the upstream app has partition * properties * @param streamDeploymentProperties deployment properties for the stream * @return true if the upstream app has partition properties */ /* default */ boolean upstreamAppHasPartitionInfo(StreamDefinition stream, StreamAppDefinition currentApp, Map<String, String> streamDeploymentProperties) { Iterator<StreamAppDefinition> iterator = stream.getDeploymentOrderIterator(); while (iterator.hasNext()) { StreamAppDefinition app = iterator.next(); if (app.equals(currentApp) && iterator.hasNext()) { StreamAppDefinition prevApp = iterator.next(); Map<String, String> appDeploymentProperties = extractAppProperties(prevApp, streamDeploymentProperties); return appDeploymentProperties.containsKey(BindingPropertyKeys.OUTPUT_PARTITION_KEY_EXPRESSION) || appDeploymentProperties .containsKey(BindingPropertyKeys.OUTPUT_PARTITION_KEY_EXTRACTOR_CLASS); } } return false; }