private boolean contentTypeChanged(Topic updated, Topic previous) { return previous.getContentType() != updated.getContentType(); }
public UnsupportedContentTypeException(Topic topic) { super(String.format( "Unsupported content type %s for topic %s", topic.getContentType(), topic.getQualifiedName() )); }
private List<Topic> getAvroTopics() { return topicsCache.getTopics().stream() .map(CachedTopic::getTopic) .filter(topic -> ContentType.AVRO == topic.getContentType()) .collect(toList()); }
@Override public void updateTopic(Topic newTopic) { if (this.topic.getContentType() != newTopic.getContentType() || messageSizeChanged(newTopic)) { logger.info("Reinitializing message receiver, contentType or messageSize changed."); this.topic = newTopic; tearDown(); initialize(); } }
@Override public void updateTopic(Topic newTopic) { if (this.topic.getContentType() != newTopic.getContentType() || messageSizeChanged(newTopic)) { logger.info("Reinitializing message receiver, contentType or messageSize changed."); this.topic = newTopic; tearDown(); initialize(); } }
@Override public String readMessageAsJson(Topic topic, KafkaTopic kafkaTopic, int partition, long offset) { byte[] bytes = kafkaRawMessageReader.readMessage(kafkaTopic, partition, offset); if (topic.getContentType() == ContentType.AVRO) { bytes = convertAvroToJson(topic, bytes); } return new String(bytes, Charset.forName("UTF-8")); }
private void updateLatestSchema(Topic topic, String operationName) { if (topic.getContentType() == ContentType.AVRO) { logger.info("Loading latest schema for {} topic {}", operationName, topic.getQualifiedName()); schemaVersionsRepository.onlineLatestSchemaVersion(topic).ifPresent( schemaVersion -> repository.getSchema(topic, schemaVersion, true)); } } });
@Override public void updateTopic(Topic newTopic) { if (this.topic.getContentType() != newTopic.getContentType() || messageSizeChanged(newTopic) || this.topic.isSchemaVersionAwareSerializationEnabled() != newTopic.isSchemaVersionAwareSerializationEnabled()) { logger.info("Reinitializing message receiver, contentType, messageSize or schemaVersionAwareSerialization changed."); this.topic = newTopic; messageReceiver.stop(); initializeMessageReceiver(); } }
public void check(Subscription subscription, Topic topic) { subscription.getFilters() .stream() .map(filter -> new ContentTypeFilterTypePair(topic.getContentType(), filter.getType())) .forEach(pair -> checkTypeMaching(pair)); }
@Override public void updateTopic(Topic newTopic) { if (this.topic.getContentType() != newTopic.getContentType() || messageSizeChanged(newTopic) || this.topic.isSchemaVersionAwareSerializationEnabled() != newTopic.isSchemaVersionAwareSerializationEnabled()) { logger.info("Reinitializing message receiver, contentType, messageSize or schemaVersionAwareSerialization changed."); this.topic = newTopic; messageReceiver.stop(); initializeMessageReceiver(); } }
@Override public void check(Message message, Topic topic) { if (ContentType.AVRO != topic.getContentType() || (ContentType.JSON == topic.getContentType() && !topic.isJsonToAvroDryRunEnabled())) { return; } BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(message.getData(), null); try { new GenericDatumReader<>(message.getSchema()).read(null, binaryDecoder); } catch (Exception e) { String reason = e.getMessage() == null ? ExceptionUtils.getRootCauseMessage(e) : e.getMessage(); throw new InvalidMessageException("Could not deserialize avro message with provided schema", ImmutableList.of(reason)); } } }
@JsonCreator public TopicWithSchema(@JsonProperty("schema") String schema, @JsonProperty("name") String qualifiedName, @JsonProperty("description") String description, @JsonProperty("owner") OwnerId owner, @JsonProperty("retentionTime") RetentionTime retentionTime, @JsonProperty("jsonToAvroDryRun") boolean jsonToAvroDryRunEnabled, @JsonProperty("ack") Ack ack, @JsonProperty("trackingEnabled") boolean trackingEnabled, @JsonProperty("migratedFromJsonType") boolean migratedFromJsonType, @JsonProperty("schemaVersionAwareSerializationEnabled") boolean schemaVersionAwareSerializationEnabled, @JsonProperty("contentType") ContentType contentType, @JsonProperty("maxMessageSize") Integer maxMessageSize, @JsonProperty("auth") PublishingAuth publishingAuth, @JsonProperty("subscribingRestricted") boolean subscribingRestricted, @JsonProperty("offlineStorage") TopicDataOfflineStorage offlineStorage) { super(qualifiedName, description, owner, retentionTime, jsonToAvroDryRunEnabled, ack, trackingEnabled, migratedFromJsonType, schemaVersionAwareSerializationEnabled, contentType, maxMessageSize, publishingAuth, subscribingRestricted, offlineStorage); this.topic = convertToTopic(); this.schema = schema; }
public TopicWithSchema getTopicWithSchema(TopicName topicName) { Topic topic = getTopicDetails(topicName); Optional<RawSchema> schema = Optional.empty(); if (AVRO.equals(topic.getContentType())) { schema = schemaService.getSchema(topicName.qualifiedName()); } return schema .map(s -> topicWithSchema(topic, s.value())) .orElseGet(() -> topicWithSchema(topic)); }
public void updateTopicWithSchema(TopicName topicName, PatchData patch, String modifiedBy) { Topic topic = getTopicDetails(topicName); boolean validateAvroSchema = AVRO.equals(topic.getContentType()); extractSchema(patch) .ifPresent(schema -> { schemaService.registerSchema(topic, schema, validateAvroSchema); scheduleTouchTopic(topicName); }); updateTopic(topicName, patch, modifiedBy); }
private void removeSchema(Topic topic) { if (AVRO.equals(topic.getContentType()) && topicProperties.isRemoveSchema()) { schemaService.getSchema(topic.getQualifiedName()).ifPresent(s -> schemaService.deleteAllSchemaVersions(topic.getQualifiedName())); } }
public void registerSchema(Topic topic, String schema, boolean validate) { if (validate) { SchemaValidator validator = validatorProvider.provide(topic.getContentType()); validator.check(schema); } rawSchemaClient.registerSchema(topic.getName(), RawSchema.valueOf(schema)); }
private Message create(HeaderMap headerMap, Topic topic, String messageId, byte[] messageContent) { long timestamp = clock.millis(); switch (topic.getContentType()) { case JSON: { if (topic.isJsonToAvroDryRunEnabled()) { try { createAvroMessage(headerMap, topic, messageId, messageContent, timestamp); } catch (AvroConversionException exception) { logger.warn("Unsuccessful message conversion from JSON to AVRO on topic {} in dry run mode", topic.getQualifiedName(), exception); } catch (WrappingException exception) { logger.warn("Unsuccessful wrapping of AVRO message on topic {} in dry run mode", topic.getQualifiedName(), exception); } } return createJsonMessage(headerMap, messageId, messageContent, timestamp); } case AVRO: return createAvroMessage(headerMap, topic, messageId, messageContent, timestamp); default: throw new UnsupportedContentTypeException(topic); } }
public void createTopicWithSchema(TopicWithSchema topicWithSchema, String createdBy, CreatorRights isAllowedToManage) { Topic topic = topicWithSchema.getTopic(); topicValidator.ensureCreatedTopicIsValid(topic, isAllowedToManage); ensureTopicDoesNotExist(topic); boolean validateAndRegisterSchema = AVRO.equals(topic.getContentType()) || (topic.isJsonToAvroDryRunEnabled() && topicWithSchema.getSchema() != null); validateSchema(validateAndRegisterSchema, topicWithSchema, topic); registerAvroSchema(validateAndRegisterSchema, topicWithSchema, createdBy); createTopic(topic, createdBy); }
public void ensureUpdatedTopicIsValid(Topic updated, Topic previous) { apiPreconditions.checkConstraints(updated); checkOwner(updated); if (migrationFromJsonTypeFlagChangedToTrue(updated, previous)) { if (updated.getContentType() != ContentType.AVRO) { throw new TopicValidationException("Change content type to AVRO together with setting migratedFromJsonType flag"); } try { schemaRepository.getLatestAvroSchema(updated); } catch (CouldNotLoadSchemaException | SchemaNotFoundException e) { throw new TopicValidationException("Avro schema not available, migration not permitted", e); } } else if (contentTypeChanged(updated, previous)) { throw new TopicValidationException("Cannot change content type, except for migration to Avro with setting migratedFromJsonType flag."); } else if (migrationFromJsonTypeFlagChangedToFalse(updated, previous)) { throw new TopicValidationException("Cannot migrate back to JSON!"); } }