private List<com.homeaway.streamplatform.streamregistry.model.Consumer> getAllConsumers(String streamName) { List<com.homeaway.streamplatform.streamregistry.model.Consumer> consumers = new ArrayList<>(); // pull data from state store of this instance. log.info("Pulling stream information from local instance's state-store for stream={} ; consumers=all", streamName); Optional<AvroStream> streamValue = kStreams.getAvroStreamForKey(AvroStreamKey.newBuilder().setStreamName(streamName).build()); if (streamValue.isPresent() && streamValue.get().getConsumers() != null) { streamValue.get().setOperationType(OperationType.GET); for (com.homeaway.digitalplatform.streamregistry.Consumer consumer : streamValue.get().getConsumers()) { consumers.add(AvroToJsonDTO.getJsonConsumer(consumer)); } } return consumers; }
public void updateAvroStream(AvroStream stream) { AvroStreamKey key = AvroStreamKey.newBuilder().setStreamName(stream.getName()).build(); stream.setOperationType(OperationType.UPSERT); try { kafkaProducer.log(key, stream); } catch (Exception e) { log.error("Error logging stream - {}", stream.toString()); } }
public void updateAvroStream(AvroStream stream) { AvroStreamKey key = AvroStreamKey.newBuilder().setStreamName(stream.getName()).build(); stream.setOperationType(OperationType.UPSERT); try { kafkaProducer.log(key, stream); } catch (Exception e) { log.error("Error logging stream - {}", stream, e); } } }
.schemaString(avroStream.getLatestKeySchema().getSchemaString()) .id(avroStream.getLatestKeySchema().getId()) .version(avroStream.getLatestKeySchema().getSubjectId()) .created(avroStream.getLatestKeySchema().getCreated()) .updated(avroStream.getLatestKeySchema().getUpdated()) .build(); .schemaString(avroStream.getLatestValueSchema().getSchemaString()) .id(avroStream.getLatestValueSchema().getId()) .version(avroStream.getLatestValueSchema().getSubjectId()) .created(avroStream.getLatestValueSchema().getCreated()) .updated(avroStream.getLatestValueSchema().getUpdated()) .build(); com.homeaway.digitalplatform.streamregistry.Tags tagsAvro = avroStream.getTags(); Tags tags = Tags.builder().productId(tagsAvro.getProductId()) .portfolioId(tagsAvro.getPortfolioId()) .name(avroStream.getName()) .schemaCompatibility(avroStream.getSchemaCompatibility()) .owner(avroStream.getOwner()) .created(avroStream.getCreated()) .updated(avroStream.getUpdated()) .githubUrl(avroStream.getGithubUrl()) .latestKeySchema(keySchema) .latestValueSchema(valueSchema) .isDataNeededAtRest(avroStream.getIsDataNeededAtRest()) .isAutomationNeeded(avroStream.getIsAutomationNeeded()) .tags(tags)
if (value.isPresent()) { if (stream.getPartitions() != value.get().getPartitions()) { throw new IllegalArgumentException(String.format( "Stream registry does not currently support modifying the initial partition count. " + "Requested: %s. Current count: %s", stream.getPartitions(), value.get().getPartitions())); } else if (stream.getReplicationFactor() != value.get().getReplicationFactor()) { throw new IllegalArgumentException(String.format( "Stream registry does not currently support modifying the initial replication factor. " + "Requested: %s. Current replication factor: %s", stream.getReplicationFactor(), value.get().getReplicationFactor())); avroStream.setCreated(value.get().getCreated()); avroStream.setProducers(value.get().getProducers()); avroStream.setConsumers(value.get().getConsumers()); avroStream.setRegionReplicatorList(value.get().getRegionReplicatorList()); avroStream.setS3ConnectorList(value.get().getS3ConnectorList()); } else { log.info("key NOT available for the stream-name={}", stream.getName()); isNewStream = true; avroStream.setCreated(System.currentTimeMillis()); key = AvroStreamKey.newBuilder().setStreamName(avroStream.getName()).build();
private Optional<com.homeaway.streamplatform.streamregistry.model.Consumer> createConsumer(AvroStream avroStream, String consumerName, String region) { log.info("==>>> getting into creating consumer. Initial Stream: {}", avroStream.toString()); if (!regionDao.getSupportedRegions(avroStream.getTags().getHint()).contains(region)) throw new UnknownRegionException(region); List<com.homeaway.digitalplatform.streamregistry.Consumer> listConsumers = avroStream.getConsumers(); if (listConsumers == null) { listConsumers = new ArrayList<>(); } com.homeaway.digitalplatform.streamregistry.Consumer consumer = com.homeaway.digitalplatform.streamregistry.Consumer .newBuilder() .setActor(Actor.newBuilder() .setName(consumerName) .build()) .build(); String streamHint = avroStream.getTags().getHint(); String hint = (streamHint == null || streamHint.trim().matches("(?i:string)?")) ? AbstractDao.PRIMARY_HINT : streamHint.trim().toLowerCase(); Actor actor = populateActorStreamConfig(avroStream.getName(), region, consumer.getActor(), OPERATION.CREATE.name(), TOPIC_POST_FIXES, hint, ACTOR_TYPE, avroStream.getTopicConfig()); consumer = Consumer.newBuilder() .setActor(actor) .build(); listConsumers.add(consumer); avroStream.setConsumers(listConsumers); updateAvroStream(avroStream); return Optional.of(AvroToJsonDTO.getJsonConsumer(consumer)); }
private Optional<com.homeaway.streamplatform.streamregistry.model.Producer> registerProducer(AvroStream avroStream, String producerName, String region) { if (!regionDao.getSupportedRegions(avroStream.getTags().getHint()).contains(region)) throw new UnknownRegionException(region); List<com.homeaway.digitalplatform.streamregistry.Producer> listProducers = avroStream.getProducers(); if (listProducers == null) { listProducers = new ArrayList<>(); } com.homeaway.digitalplatform.streamregistry.Producer producer = com.homeaway.digitalplatform.streamregistry.Producer.newBuilder() .setActor(Actor.newBuilder() .setName(producerName) .build()) .build(); String streamHint = avroStream.getTags().getHint(); String hint = (streamHint == null || streamHint.trim().matches("(?i:string)?")) ? AbstractDao.PRIMARY_HINT : streamHint.trim().toLowerCase(); Actor actor = populateActorStreamConfig(avroStream.getName(), region, producer.getActor(), OPERATION.CREATE.name(), topicPostFixes, hint, ACTOR_TYPE, avroStream.getTopicConfig()); Producer newProducer = com.homeaway.digitalplatform.streamregistry.Producer.newBuilder() .setActor(actor) .build(); listProducers.add(newProducer); avroStream.setProducers(listProducers); updateAvroStream(avroStream); return Optional.of(AvroToJsonDTO.getJsonProducer(newProducer)); }
List<Consumer> consumers = avroStream.get().getConsumers(); if (consumers != null) { for (com.homeaway.digitalplatform.streamregistry.Consumer consumer : consumers) { if (streamConfiguration.getRegion().equals(region)) { String streamHint = avroStream.get().getTags().getHint(); String hint = (streamHint == null || streamHint.trim().matches("(?i:string)?")) ? AbstractDao.PRIMARY_HINT : streamHint.trim().toLowerCase(); Actor consumerActor = populateActorStreamConfig(streamName, region, consumer.getActor(), OPERATION.UPDATE.name(), TOPIC_POST_FIXES, hint, ACTOR_TYPE, avroStream.get().getTopicConfig()); consumer.setActor(consumerActor); updateAvroStream(avroStream.get());
private void deleteConsumer(String streamName, String consumerName) { Optional<AvroStream> avroStream = getAvroStreamKeyValue(streamName).getValue(); if (avroStream.isPresent()) { final List<com.homeaway.digitalplatform.streamregistry.Consumer> withConsumer = avroStream.get().getConsumers(); // Obtains consumer list size before remove consumer final int consumerInitialSize = withConsumer.size(); // Obtains filtered consumer list not containing the consumer we want to remove List<com.homeaway.digitalplatform.streamregistry.Consumer> withoutConsumer = withConsumer .stream() .filter(consumer -> !StreamRegistryUtils.hasActorNamed(consumerName, consumer::getActor)) .collect(Collectors.toList()); // Update stream's consumer list avroStream.get().setConsumers(withoutConsumer); // If filtered consumer list size is less than initial size stream will be updated if (avroStream.get().getConsumers().size() < consumerInitialSize) updateAvroStream(avroStream.get()); else throw new ConsumerNotFoundException(consumerName); } else { throw new StreamNotFoundException(streamName); } } }
private void validateStateStore() { try { AvroStreamKey avroStreamKey = AvroStreamKey.newBuilder().setStreamName(HEALTH_CHECK_STREAM_NAME).build(); Optional<AvroStream> avroStreamValue = managedKStreams.getAvroStreamForKey(avroStreamKey); if(!avroStreamValue.isPresent() || ! avroStreamValue.get().getName().equals(HEALTH_CHECK_STREAM_NAME)) { setStateStoreHealthy(false); throw new IllegalStateException("HealthCheck Failed: StreamRegistryHealthCheck Stream not available in StateStore."); } } catch (Exception e) { setStateStoreHealthy(false); throw e; } setStateStoreHealthy(true); }
public AvroStream build() { try { AvroStream record = new AvroStream(); record.name = fieldSetFlags()[0] ? this.name : (java.lang.String) defaultValue(fields()[0]); record.schemaCompatibility = fieldSetFlags()[1] ? this.schemaCompatibility : (com.homeaway.digitalplatform.streamregistry.SchemaCompatibility) defaultValue(fields()[1]);
private Optional<com.homeaway.streamplatform.streamregistry.model.Consumer> getConsumer(String streamName, String consumerName) { // pull data from state store of this instance. log.info("Pulling stream information from local instance's state-store for streamName={} ; consumerName={}", streamName, consumerName); Optional<AvroStream> streamValue = kStreams.getAvroStreamForKey( AvroStreamKey.newBuilder().setStreamName(streamName).build()); if (streamValue.isPresent()) { streamValue.get().setOperationType(OperationType.GET); for (com.homeaway.digitalplatform.streamregistry.Consumer consumer : streamValue.get().getConsumers()) { if (consumer.getActor().getName().equals(consumerName)) return Optional.of(AvroToJsonDTO.getJsonConsumer(consumer)); } } return Optional.empty(); }