private String getIdForSchema(Schema schema) { if (!this.registrySchemaIds.containsKey(schema)) { try { String schemaId = this.overrideName.isPresent() ? this.registry.register(schema, this.overrideName.get()) : this.registry.register(schema); this.registrySchemaIds.put(schema, schemaId); } catch (SchemaRegistryException e) { throw Throwables.propagate(e); } } return this.registrySchemaIds.get(schema); }
public KafkaSchemaRegistry create(Properties props) { return new KafkaAvroSchemaRegistry(props); } }
HttpClient httpClient = this.borrowClient(); int loop = 0;
public SchemaRegistryVersionWriter(KafkaAvroSchemaRegistry registry, Optional<String> overrideName, Optional<Schema> singleSchema) throws IOException { this.registry = registry; this.registrySchemaIds = Maps.newConcurrentMap(); this.overrideName = overrideName; this.schema = singleSchema; this.schemaIdLengthBytes = registry.getSchemaIdLengthByte(); if (this.schema.isPresent()) { try { this.schemaId = this.overrideName.isPresent() ? Optional .of(this.registry.register(this.schema.get(), this.overrideName.get())) : Optional.of(this.registry.register(this.schema.get())); } catch (SchemaRegistryException e) { throw Throwables.propagate(e); } } else { this.schemaId = Optional.absent(); } }
@Override public Schema readSchemaVersioningInformation(DataInputStream inputStream) throws IOException { if (inputStream.readByte() != KafkaAvroSchemaRegistry.MAGIC_BYTE) { throw new IOException("MAGIC_BYTE not found in Avro message."); } byte[] byteKey = new byte[schemaIdLengthBytes]; int bytesRead = inputStream.read(byteKey, 0, schemaIdLengthBytes); if (bytesRead != schemaIdLengthBytes) { throw new IOException(String .format("Could not read enough bytes for schema id. Expected: %d, found: %d.", schemaIdLengthBytes, bytesRead)); } String hexKey = Hex.encodeHexString(byteKey); try { return this.registry.getSchemaByKey(hexKey); } catch (SchemaRegistryException sre) { throw new IOException("Failed to retrieve schema for key " + hexKey, sre); } } }
public SchemaRegistryVersionWriter(KafkaAvroSchemaRegistry registry, Optional<String> overrideName, Optional<Schema> singleSchema) throws IOException { this.registry = registry; this.registrySchemaIds = Maps.newConcurrentMap(); this.overrideName = overrideName; this.schema = singleSchema; this.schemaIdLengthBytes = registry.getSchemaIdLengthByte(); if (this.schema.isPresent()) { try { this.schemaId = this.overrideName.isPresent() ? Optional .of(this.registry.register(this.schema.get(), this.overrideName.get())) : Optional.of(this.registry.register(this.schema.get())); } catch (SchemaRegistryException e) { throw Throwables.propagate(e); } } else { this.schemaId = Optional.absent(); } }
@Override public Schema readSchemaVersioningInformation(DataInputStream inputStream) throws IOException { if (inputStream.readByte() != KafkaAvroSchemaRegistry.MAGIC_BYTE) { throw new IOException("MAGIC_BYTE not found in Avro message."); } byte[] byteKey = new byte[schemaIdLengthBytes]; int bytesRead = inputStream.read(byteKey, 0, schemaIdLengthBytes); if (bytesRead != schemaIdLengthBytes) { throw new IOException(String .format("Could not read enough bytes for schema id. Expected: %d, found: %d.", schemaIdLengthBytes, bytesRead)); } String hexKey = Hex.encodeHexString(byteKey); try { return this.registry.getSchemaByKey(hexKey); } catch (SchemaRegistryException sre) { throw new IOException("Failed to retrieve schema for key " + hexKey, sre); } } }
/** * Register a schema to the Kafka schema registry under the provided input name. This method will change the name * of the schema to the provided name. This is useful because certain services (like Gobblin kafka adaptor and * Camus) get the schema for a topic by querying for the latest schema with the topic name, requiring the topic * name and schema name to match for all topics. This method registers the schema to the schema registry in such a * way that any schema can be written to any topic. * * @param schema {@link org.apache.avro.Schema} to register. * @param overrideName Name of the schema when registerd to the schema registry. This name should match the name * of the topic where instances will be published. * @return schema ID of the registered schema. * @throws SchemaRegistryException if registration failed */ @Override public String register(Schema schema, String overrideName) throws SchemaRegistryException { return register(AvroUtils.switchName(schema, overrideName)); }
public SchemaRegistryVersionWriter(Config config) throws IOException { this(new KafkaAvroSchemaRegistry(ConfigUtils.configToProperties(config)), Optional.<String>absent(), Optional.<Schema>absent()); }
HttpClient httpClient = this.borrowClient(); try { statusCode = httpClient.executeMethod(get);
private static String getSchemaId(Properties properties, Schema schema) throws SchemaRegistryException { KafkaAvroSchemaRegistry kafkaAvroSchemaRegistry = (KafkaAvroSchemaRegistry) new KafkaAvroSchemaRegistryFactory().create(properties); return kafkaAvroSchemaRegistry.register(schema); }
/** * Get a {@link org.apache.gobblin.metrics.kafka.KafkaReporter.Builder} for this reporting format. * * @param properties {@link Properties} containing information to build reporters. * @return {@link org.apache.gobblin.metrics.kafka.KafkaReporter.Builder}. */ public KafkaReporter.Builder<?> metricReporterBuilder(Properties properties) { switch (this) { case AVRO: KafkaAvroReporter.Builder<?> builder = KafkaAvroReporter.BuilderFactory.newBuilder(); if (Boolean.valueOf(properties.getProperty(ConfigurationKeys.METRICS_REPORTING_KAFKA_USE_SCHEMA_REGISTRY, ConfigurationKeys.DEFAULT_METRICS_REPORTING_KAFKA_USE_SCHEMA_REGISTRY))) { builder.withSchemaRegistry(new KafkaAvroSchemaRegistry(properties)); } return builder; case JSON: return KafkaReporter.BuilderFactory.newBuilder(); default: // This should never happen. throw new IllegalArgumentException("KafkaReportingFormat not recognized."); } }
post.addParameter("schema", schema.toString()); HttpClient httpClient = this.borrowClient(); try { LOG.debug("Loading: " + post.getURI());
Mockito.when(registry.register(Mockito.any(Schema.class))).thenAnswer(new Answer<String>() { @Override public String answer(InvocationOnMock invocation) Mockito.when(registry.register(Mockito.any(Schema.class), Mockito.anyString())).thenAnswer(new Answer<String>() { @Override public String answer(InvocationOnMock invocation)
if (Boolean.valueOf(properties.getProperty(ConfigurationKeys.METRICS_REPORTING_KAFKA_USE_SCHEMA_REGISTRY, ConfigurationKeys.DEFAULT_METRICS_REPORTING_KAFKA_USE_SCHEMA_REGISTRY))) { kafkaAvroEventReporterBuilder.withSchemaRegistry(new KafkaAvroSchemaRegistry(properties)); kafkaAvroEventKeyValueReporterBuilder.withSchemaRegistry(new KafkaAvroSchemaRegistry(properties));
HttpClient httpClient = this.borrowClient(); int loop = 0;
private String getIdForSchema(Schema schema) { if (!this.registrySchemaIds.containsKey(schema)) { try { String schemaId = this.overrideName.isPresent() ? this.registry.register(schema, this.overrideName.get()) : this.registry.register(schema); this.registrySchemaIds.put(schema, schemaId); } catch (SchemaRegistryException e) { throw Throwables.propagate(e); } } return this.registrySchemaIds.get(schema); }
public KafkaSchemaRegistry create(Properties props) { return new KafkaAvroSchemaRegistry(props); } }
HttpClient httpClient = this.borrowClient(); try { statusCode = httpClient.executeMethod(get);
/** * Register a schema to the Kafka schema registry under the provided input name. This method will change the name * of the schema to the provided name. This is useful because certain services (like Gobblin kafka adaptor and * Camus) get the schema for a topic by querying for the latest schema with the topic name, requiring the topic * name and schema name to match for all topics. This method registers the schema to the schema registry in such a * way that any schema can be written to any topic. * * @param schema {@link org.apache.avro.Schema} to register. * @param overrideName Name of the schema when registerd to the schema registry. This name should match the name * of the topic where instances will be published. * @return schema ID of the registered schema. * @throws SchemaRegistryException if registration failed */ @Override public String register(Schema schema, String overrideName) throws SchemaRegistryException { return register(AvroUtils.switchName(schema, overrideName)); }