@Override public int hashCode() { int result = getStreamId().hashCode(); result = 31 * result + (getSchema() != null ? getSchema().hashCode() : 0); result = 31 * result + (getSinkConfig() != null ? getSinkConfig().hashCode() : 0); result = 31 * result + (getSourceConfig() != null ? getSourceConfig().hashCode() : 0); return result; } }
@Override public void open(StreamDesc streamDesc) { if (streamDesc.getSchema() != null) { this.validator = new StreamValidator(streamDesc.getSchema()); } this.streamId = streamDesc.getStreamId(); if (streamDesc.getSinkConfig() != null) { this.proxyProducer = new KafkaStreamProxyProducerImpl(streamDesc.getStreamId(), streamDesc.getSinkConfig()); } else { LOGGER.warn("Unable to initialize kafka producer because sink config is null for {}", streamId); this.proxyProducer = null; } this.opened = true; }
StreamDesc streamDesc = new StreamDesc(); streamDesc.setSchema(copied); streamDesc.setSinkConfig(streamSinkConfig); streamDesc.setSourceConfig(streamSourceConfig); streamDesc.setStreamId(copied.getStreamId()); if (streamDesc.getSinkConfig() instanceof KafkaStreamSinkConfig) { KafkaStreamSinkConfig kafkaCfg = (KafkaStreamSinkConfig) streamDesc.getSinkConfig(); Kafka2TupleMetadata datasource = new Kafka2TupleMetadata(); datasource.setType("KAFKA"); datasource.setName(streamDesc.getStreamId()); datasource.setTopic(kafkaCfg.getTopicId()); datasource.setSchemeCls(JsonScheme.class.getCanonicalName()); datasource.setProperties(new HashMap<>()); KafkaStreamSourceConfig streamSourceConfig = (KafkaStreamSourceConfig) streamDesc.getSourceConfig(); if (streamSourceConfig != null) { Map<String, String> properties = datasource.getProperties(); prop.put(JsonStringStreamNameSelector.USER_PROVIDED_STREAM_NAME_PROPERTY, streamDesc.getStreamId()); tuple2Stream.setStreamNameSelectorProp(prop); tuple2Stream.setTimestampColumn("timestamp"); StreamDefinition sd = streamDesc.getSchema(); sd.setDataSource(streamDesc.getStreamId()); sd.setStreamSource(metadata.getAppId()); alertMetadataService.createStream(streamDesc.getSchema());
private void fillApplicationDesc(List<ApplicationEntity> entities) { for (ApplicationEntity entity : entities) { entity.setDescriptor(applicationProviderService.getApplicationDescByType(entity.getDescriptor().getType())); if (entity.getDescriptor().getStreams() == null) { continue; } List<StreamDesc> streamDescToInstall = entity.getDescriptor().getStreams().stream().map((streamDefinition -> { StreamDefinition copied = streamDefinition.copy(); copied.setSiteId(entity.getSite().getSiteId()); copied.setStreamId(StreamIdConversions.formatSiteStreamId(entity.getSite().getSiteId(), copied.getStreamId())); Config effectiveConfig = ConfigFactory.parseMap(new HashMap<>(entity.getConfiguration())) .withFallback(config).withFallback(ConfigFactory.parseMap(entity.getContext())); ExecutionRuntime runtime = ExecutionRuntimeManager.getInstance().getRuntime( applicationProviderService.getApplicationProviderByType(entity.getDescriptor().getType()).getApplication().getEnvironmentType(), config); StreamSinkConfig streamSinkConfig = runtime.environment() .stream().getSinkConfig(StreamIdConversions.parseStreamTypeId(copied.getSiteId(), copied.getStreamId()), effectiveConfig); StreamDesc streamDesc = new StreamDesc(); streamDesc.setSchema(copied); streamDesc.setSinkConfig(streamSinkConfig); streamDesc.setStreamId(copied.getStreamId()); streamDesc.getSchema().setDataSource(entity.getAppId()); return streamDesc; })).collect(Collectors.toList()); entity.setStreams(streamDescToInstall); } }
@Override public void onStreamRemoved(StreamDesc streamDesc) { LOGGER.info("Removing stream proxy {}", streamDesc.getStreamId()); if (streamProxyConcurrentMap.containsKey(streamDesc.getStreamId())) { try { streamProxyConcurrentMap.get(streamDesc.getStreamId()).close(); } catch (IOException e) { LOGGER.error("Unable to close {}", streamProxyConcurrentMap.get(streamDesc.getStreamId())); } } else { LOGGER.warn("Unable to remove stream proxy {}, because not exist", streamDesc.getStreamId()); } } }
for (StreamDesc streamDesc : streamDescList) { total++; latestStreamIdDescMap.put(streamDesc.getStreamId(), streamDesc); if (streamIdDescMap.containsKey(streamDesc.getStreamId()) && !streamDesc.equals(streamIdDescMap.get(streamDesc.getStreamId()))) { changed++; this.listener.onStreamChanged(streamDesc); } else if (!streamIdDescMap.containsKey(streamDesc.getStreamId())) { added++; this.listener.onStreamAdded(streamDesc);
@Override public void onStreamAdded(StreamDesc streamDesc) { if (streamProxyConcurrentMap.containsKey(streamDesc.getStreamId())) { LOGGER.warn("Adding already existing stream proxy {}", streamDesc.getStreamId()); this.onStreamChanged(streamDesc); return; } LOGGER.info("Adding stream proxy {}", streamDesc.getStreamId()); StreamProxy proxy = new StreamProxyImpl(); proxy.open(streamDesc); streamProxyConcurrentMap.put(streamDesc.getStreamId(), proxy); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof StreamDesc)) { return false; } StreamDesc that = (StreamDesc) o; if (!getStreamId().equals(that.getStreamId())) { return false; } if (getSchema() != null ? !getSchema().equals(that.getSchema()) : that.getSchema() != null) { return false; } if (getSinkConfig() != null ? !getSinkConfig().equals(that.getSinkConfig()) : that.getSinkConfig() != null) { return false; } return getSourceConfig() != null ? getSourceConfig().equals(that.getSourceConfig()) : that.getSourceConfig() == null; }
@Override public void onStreamChanged(StreamDesc streamDesc) { if (!streamProxyConcurrentMap.containsKey(streamDesc.getStreamId())) { LOGGER.warn("Updating non-existing stream proxy {}", streamDesc.getStreamId()); this.onStreamAdded(streamDesc); return; } LOGGER.info("Updating stream proxy {}", streamDesc.getStreamId()); try { LOGGER.info("Closing old stream proxy {}", streamDesc.getStreamId()); streamProxyConcurrentMap.get(streamDesc.getStreamId()).close(); } catch (IOException e) { LOGGER.error("Unable to close {}", streamProxyConcurrentMap.get(streamDesc.getStreamId())); } finally { LOGGER.info("Adding stream proxy {}", streamDesc.getStreamId()); StreamProxyImpl proxy = new StreamProxyImpl(); proxy.open(streamDesc); streamProxyConcurrentMap.put(streamDesc.getStreamId(), proxy); } }
public void doUninstall() { // we should remove alert data source and stream definition while we do uninstall if (metadata.getStreams() == null) { return; } // iterate each stream descriptor and create alert datasource for each for (StreamDesc streamDesc : metadata.getStreams()) { alertMetadataService.removeDataSource(streamDesc.getStreamId()); alertMetadataService.removeStream(streamDesc.getStreamId()); } }
@GET @Path("/{streamId}") @Produces(MediaType.APPLICATION_JSON) public RESTResponse getSingleStreamDesc(@PathParam("streamId") String streamId) { return RESTResponse.async((builder) -> { Optional<StreamDesc> streamDesc = proxyManager.getAllStreamDesc() .stream().filter((desc) -> desc.getStreamId().equalsIgnoreCase(streamId)).findAny(); if (streamDesc.isPresent()) { builder.data(streamDesc.get()) .status(true, Response.Status.OK); } else { builder.message("Stream not found, reason: stream not exist or proxy not initialized").status(false, Response.Status.BAD_REQUEST); } }).get(); } }