/** * Return the ordered list of application definitions for this stream as a * {@link List}. This allows for retrieval of application definitions in the stream by * index. Application definitions are maintained in stream flow order (source is * first, sink is last). * * @return list of application definitions for this stream definition */ public LinkedList<StreamAppDefinition> getAppDefinitions() { if (CollectionUtils.isEmpty(this.applicationDefinitions)) { return getAppDefinitions(this.name, this.dslText); } return this.applicationDefinitions; }
/** * Construct a {@code StreamDefinition}. * * @param name name of stream * @param dslText DSL definition for stream */ public StreamDefinition(String name, String dslText) { Assert.hasText(name, "name is required"); Assert.hasText(dslText, "dslText is required"); this.name = name; this.dslText = dslText; this.applicationDefinitions = getAppDefinitions(name, dslText); }
@PostLoad public void initialize() { if (CollectionUtils.isEmpty(this.applicationDefinitions)) { this.applicationDefinitions = getAppDefinitions(this.name, this.dslText); } }
/** * Reverse engineers a {@link StreamDefinition} into a semantically equivalent DSL text representation. * @param streamDefinition stream to be converted into DSL * @return the textual DSL representation of the stream */ public String toDsl(StreamDefinition streamDefinition) { return toDsl(streamDefinition.getAppDefinitions()); }
@Override public Map<StreamDefinition, DeploymentState> streamsStates(List<StreamDefinition> streamDefinitions) { Map<StreamDefinition, List<String>> deploymentIdsPerStream = streamDefinitions.stream() .collect(Collectors.toMap(Function.identity(), sd -> sd.getAppDefinitions().stream().map( sad -> deploymentIdRepository.findOne(DeploymentKey.forStreamAppDefinition(sad))) .collect(Collectors.toList()))); // Map from app deployment id to DeploymentState Map<String, DeploymentState> statePerApp = gatherDeploymentStates(deploymentIdsPerStream.values().stream() .flatMap(Collection::stream).filter(Objects::nonNull).toArray(String[]::new)); // Map from SCDF Stream to aggregate streamsStates return deploymentIdsPerStream.entrySet().stream() .map(kv -> new AbstractMap.SimpleImmutableEntry<>(kv.getKey(), StreamDeployerUtil.aggregateState(kv.getValue().stream() .map(deploymentId -> statePerApp.getOrDefault(deploymentId, DeploymentState.unknown)) .collect(Collectors.toSet())))) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); }
private String getRegisteredName(StreamDefinition streamDefinition, String adrAppName) { for (StreamAppDefinition appDefinition: streamDefinition.getAppDefinitions()) { if (appDefinition.getName().equals(adrAppName)) { return appDefinition.getRegisteredAppName(); } } return adrAppName; }
private String getRegisteredName(StreamDefinition streamDefinition, String adrAppName) { for (StreamAppDefinition appDefinition : streamDefinition.getAppDefinitions()) { if (appDefinition.getName().equals(adrAppName)) { return appDefinition.getRegisteredAppName(); } } return adrAppName; }
/** * Return an iterator that indicates the order of application deployments for this * stream. The application definitions are returned in reverse order; i.e. the sink is * returned first followed by the processors in reverse order followed by the source. * * @return iterator that iterates over the application definitions in deployment order */ public Iterator<StreamAppDefinition> getDeploymentOrderIterator() { return new ReadOnlyIterator<>(getAppDefinitions().descendingIterator()); }
@Override public boolean addProposals(String text, StreamDefinition streamDefinition, int detailLevel, List<CompletionProposal> collector) { StreamAppDefinition lastApp = streamDefinition.getDeploymentOrderIterator().next(); AppRegistration appRegistration = this.collectorSupport.findAppRegistration(lastApp.getName(), CompletionUtils.determinePotentialTypes(lastApp,streamDefinition.getAppDefinitions().size() > 1)); if (appRegistration != null) { Set<String> alreadyPresentOptions = new HashSet<>(lastApp.getProperties().keySet()); this.collectorSupport.addPropertiesProposals(text, "", appRegistration, alreadyPresentOptions, collector, detailLevel); } return false; } }
@Override public boolean addProposals(String text, StreamDefinition parseResult, int detailLevel, List<CompletionProposal> collector) { Set<String> propertyNames = new HashSet<>( parseResult.getDeploymentOrderIterator().next().getProperties().keySet()); propertyNames.removeAll(CompletionUtils.IMPLICIT_PARAMETER_NAMES); if (text.endsWith(" ") || propertyNames.isEmpty()) { return false; } String propertyName = recoverPropertyName(text); StreamAppDefinition lastApp = parseResult.getDeploymentOrderIterator().next(); String alreadyTyped = lastApp.getProperties().get(propertyName); AppRegistration lastAppRegistration = this.collectorSupport.findAppRegistration(lastApp.getName(), CompletionUtils.determinePotentialTypes(lastApp, parseResult.getAppDefinitions().size() > 1)); if (lastAppRegistration != null) { return this.collectorSupport.addAlreadyTypedValueHintsProposals(text, lastAppRegistration, collector, propertyName, valueHintProviders, alreadyTyped); } return false; }
@Override public void addProposals(String dsl, CheckPointedParseException exception, int detailLevel, List<CompletionProposal> collector) { String safe = exception.getExpressionStringUntilCheckpoint(); StreamDefinition streamDefinition = new StreamDefinition("__dummy", safe); StreamAppDefinition lastApp = streamDefinition.getDeploymentOrderIterator().next(); AppRegistration appRegistration = this.collectorSupport.findAppRegistration(lastApp.getName(), CompletionUtils.determinePotentialTypes(lastApp, streamDefinition.getAppDefinitions().size() > 1)); if (appRegistration != null) { Set<String> alreadyPresentOptions = new HashSet<>(lastApp.getProperties().keySet()); this.collectorSupport.addPropertiesProposals(safe, "", appRegistration, alreadyPresentOptions, collector, detailLevel); } } }
private AppRegistration lookupLastApp(CheckPointedParseException exception) { String safe = exception.getExpressionStringUntilCheckpoint(); StreamDefinition streamDefinition = new StreamDefinition("__dummy", safe); StreamAppDefinition lastApp = streamDefinition.getDeploymentOrderIterator().next(); return this.collectorSupport.findAppRegistration(lastApp.getName(), CompletionUtils.determinePotentialTypes(lastApp, streamDefinition.getAppDefinitions().size() > 1)); }
/** * Redacts sensitive property values in a stream. * * @param streamDefinition the stream definition to sanitize * @return Stream definition text that has sensitive data redacted. */ public String sanitizeStream(StreamDefinition streamDefinition) { List<StreamAppDefinition> sanitizedAppDefinitions = streamDefinition.getAppDefinitions().stream() .map(app -> StreamAppDefinition.Builder .from(app) .setProperties(this.sanitizeProperties(app.getProperties())) .build(streamDefinition.getName()) ).collect(Collectors.toList()); return this.streamDslConverter.toDsl(sanitizedAppDefinitions); }
@Override public DeploymentState streamState(String streamName) { Set<DeploymentState> appStates = EnumSet.noneOf(DeploymentState.class); StreamDefinition stream = this.streamDefinitionRepository.findOne(streamName); for (StreamAppDefinition appDefinition : stream.getAppDefinitions()) { String key = DeploymentKey.forStreamAppDefinition(appDefinition); String id = this.deploymentIdRepository.findOne(key); if (id != null) { AppStatus status = this.appDeployer.status(id); appStates.add(status.getState()); } else { appStates.add(DeploymentState.undeployed); } } return StreamDeployerUtil.aggregateState(appStates); }
@Override public void addProposals(String dsl, CheckPointedParseException exception, int detailLevel, List<CompletionProposal> collector) { String safe = exception.getExpressionStringUntilCheckpoint(); StreamDefinition streamDefinition = new StreamDefinition("__dummy", safe); StreamAppDefinition lastApp = streamDefinition.getDeploymentOrderIterator().next(); AppRegistration appRegistration = this.collectorSupport.findAppRegistration(lastApp.getName(), CompletionUtils.determinePotentialTypes(lastApp, streamDefinition.getAppDefinitions().size() > 1)); if (appRegistration != null) { String startsWith = ProposalsCollectorSupportUtils.computeStartsWith(exception); Set<String> alreadyPresentOptions = new HashSet<>(lastApp.getProperties().keySet()); this.collectorSupport.addPropertiesProposals(safe, startsWith, appRegistration, alreadyPresentOptions, collector, detailLevel); } } }
/** * Redacts sensitive property values in a stream. * * @param streamDefinition the stream definition to sanitize * @return Stream definition text that has sensitive data redacted. */ public String sanitizeStream(StreamDefinition streamDefinition) { List<StreamAppDefinition> sanitizedAppDefinitions = streamDefinition.getAppDefinitions().stream() .map(app -> StreamAppDefinition.Builder .from(app) .setProperties(this.sanitizeProperties(app.getProperties())) .build(streamDefinition.getName()) ).collect(Collectors.toList()); return this.streamDslConverter.toDsl(sanitizedAppDefinitions); }
public StreamDefinition createStream(String streamName, String dsl, boolean deploy) { StreamDefinition streamDefinition = createStreamDefinition(streamName, dsl); List<String> errorMessages = new ArrayList<>(); for (StreamAppDefinition streamAppDefinition : streamDefinition.getAppDefinitions()) { final String appName = streamAppDefinition.getRegisteredAppName(); ApplicationType applicationType = streamAppDefinition.getApplicationType(); if (!streamValidationService.isRegistered(appName, applicationType)) { errorMessages.add( String.format("Application name '%s' with type '%s' does not exist in the app registry.", appName, applicationType)); } } if (!errorMessages.isEmpty()) { throw new InvalidStreamDefinitionException( StringUtils.collectionToDelimitedString(errorMessages, "\n")); } final StreamDefinition savedStreamDefintion = this.streamDefinitionRepository.save(streamDefinition); if (deploy) { this.deployStream(streamName, new HashMap<>()); } auditRecordService.populateAndSaveAuditRecord( AuditOperationType.STREAM, AuditActionType.CREATE, streamDefinition.getName(), this.auditServiceUtils.convertStreamDefinitionToAuditData(savedStreamDefintion)); return streamDefinition; }
public void undeployStream(String streamName) { StreamDefinition streamDefinition = this.streamDefinitionRepository.findOne(streamName); for (StreamAppDefinition appDefinition : streamDefinition.getAppDefinitions()) { String key = DeploymentKey.forStreamAppDefinition(appDefinition); String id = this.deploymentIdRepository.findOne(key); // if id is null, assume nothing is deployed if (id != null) { AppStatus status = this.appDeployer.status(id); if (!EnumSet.of(DeploymentState.unknown, DeploymentState.undeployed).contains(status.getState())) { this.appDeployer.undeploy(id); } this.deploymentIdRepository.delete(key); } } this.streamDeploymentRepository.delete(streamDefinition.getName()); }
@Override public ValidationStatus validateStream(String name) { StreamDefinition definition = streamDefinitionRepository.findById(name) .orElseThrow(() -> new NoSuchStreamDefinitionException(name)); ValidationStatus validationStatus = new ValidationStatus( definition.getName(), definition.getDslText()); for (StreamAppDefinition streamAppDefinition : definition.getAppDefinitions()) { ApplicationType appType = streamAppDefinition.getApplicationType(); boolean status = this.validate(streamAppDefinition.getName(), appType); validationStatus.getAppsStatuses().put( String.format("%s:%s", appType.name(), streamAppDefinition.getName()), (status) ? NodeStatus.valid.name() : NodeStatus.invalid.name()); } return validationStatus; } }
@Override public ValidationStatus validateStream(String name) { StreamDefinition definition = streamDefinitionRepository.findOne(name); if (definition == null) { throw new NoSuchStreamDefinitionException(name); } ValidationStatus validationStatus = new ValidationStatus( definition.getName(), definition.getDslText()); for (StreamAppDefinition streamAppDefinition : definition.getAppDefinitions()) { ApplicationType appType = streamAppDefinition.getApplicationType(); boolean status = this.validate(streamAppDefinition.getName(), appType); validationStatus.getAppsStatuses().put( String.format("%s:%s", appType.name(), streamAppDefinition.getName()), (status) ? NodeStatus.valid.name() : NodeStatus.invalid.name()); } return validationStatus; } }