public MessageDecoderHelper invoke(String id) { this.schema = (this.registry.getSchemaByID(this.topicName, id)); if (this.schema == null) throw new IllegalStateException("Unknown schema id: " + id); this.targetSchema = JSONToAvroMessageDecoder.this.latestSchema; return this; } }
@Override public void init(Properties props, String topicName) { super.init(props, topicName); try { SchemaRegistry<Schema> registry = (SchemaRegistry<Schema>) Class.forName( props.getProperty(KafkaAvroMessageEncoder.KAFKA_MESSAGE_CODER_SCHEMA_REGISTRY_CLASS)).newInstance(); log.info("Prop " + KafkaAvroMessageEncoder.KAFKA_MESSAGE_CODER_SCHEMA_REGISTRY_CLASS + " is: " + props.getProperty(KafkaAvroMessageEncoder.KAFKA_MESSAGE_CODER_SCHEMA_REGISTRY_CLASS)); log.info("Underlying schema registry for topic: " + topicName + " is: " + registry); registry.init(props); this.registry = new CachedSchemaRegistry<Schema>(registry, props); this.latestSchema = registry.getLatestSchemaByTopic(topicName).getSchema(); } catch (Exception e) { throw new MessageDecoderException(e); } decoderFactory = DecoderFactory.get(); }
public String register(String topic, S schema) { return registry.register(topic, schema); }
@Test public void testSchemaRegistry() { try { registry.getLatestSchemaByTopic("test"); fail("Should have failed with a SchemaNotFoundException."); } catch (SchemaNotFoundException e) { registry.getSchemaByID("test", "abc"); fail("Should have failed with a SchemaNotFoundException."); } catch (SchemaNotFoundException e) { String id = registry.register("test", getSchema1()); registry.getSchemaByID("test", "abc"); fail("Should have failed with a SchemaNotFoundException."); } catch (SchemaNotFoundException e) { assertEquals(getSchema1(), registry.getSchemaByID("test", id)); assertEquals(new SchemaDetails("test", id, getSchema1()), registry.getLatestSchemaByTopic("test")); String secondId = registry.register("test", getSchema2()); assertEquals(getSchema1(), registry.getSchemaByID("test", id)); assertEquals(getSchema2(), registry.getSchemaByID("test", secondId)); assertEquals(new SchemaDetails("test", secondId, getSchema2()), registry.getLatestSchemaByTopic("test")); registry.getLatestSchemaByTopic("test-2"); fail("Should have failed with a SchemaNotFoundException."); } catch (SchemaNotFoundException e) {
public SchemaDetails<S> getLatestSchemaByTopic(String topicName) { SchemaDetails<S> schema = cachedLatest.get(topicName); if (schema == null) { schema = registry.getLatestSchemaByTopic(topicName); cachedLatest.putIfAbsent(topicName, schema); } return schema; }
@Override public void init(Properties props, String topicName) { super.init(props, topicName); try { client = (SchemaRegistry<Schema>) Class.forName(props.getProperty(KAFKA_MESSAGE_CODER_SCHEMA_REGISTRY_CLASS)) .newInstance(); client.init(props); } catch (Exception e) { throw new MessageEncoderException(e); } }
@Override public CamusWrapper<Record> decode(Message message) { try { GenericDatumReader<Record> reader = new GenericDatumReader<Record>(); Schema schema = super.registry.getLatestSchemaByTopic(super.topicName).getSchema(); reader.setSchema(schema); return new CamusWrapper<Record>(reader.read(null, decoderFactory.jsonDecoder(schema, new String(message.getPayload(), //Message.payloadOffset(message.magic()), kafka.message.Message.MagicOffset(), message.getPayload().length - kafka.message.Message.MagicOffset())))); } catch (Exception e) { throw new RuntimeException(e); } } }
@Parameters public static Collection data() { Properties props = new Properties(); props.put(AvroRestSchemaRegistry.ETL_SCHEMA_REGISTRY_URL, "http://localhost:8123/schema-repo/"); SchemaRegistry<Schema> avroSchemaRegistry = new AvroRestSchemaRegistry(); avroSchemaRegistry.init(props); Object[][] data = new Object[][] { { avroSchemaRegistry } }; return Arrays.asList(data); } }
public MessageDecoderHelper invoke() { buffer = getByteBuffer(payload); String id = Integer.toString(buffer.getInt()); schema = registry.getSchemaByID(topicName, id); if (schema == null) throw new IllegalStateException("Unknown schema id: " + id); start = buffer.position() + buffer.arrayOffset(); length = buffer.limit() - 5; // try to get a target schema, if any targetSchema = latestSchema; return this; } }
public void init(Properties props, String topicName) { super.init(props, topicName); this.props = props; this.topicName = topicName; this.schemaIDField = props.getProperty(CAMUS_SCHEMA_ID_FIELD, DEFAULT_SCHEMA_ID_FIELD); try { SchemaRegistry<Schema> registry = (SchemaRegistry<Schema>) Class.forName( props.getProperty(KafkaAvroMessageEncoder.KAFKA_MESSAGE_CODER_SCHEMA_REGISTRY_CLASS)).newInstance(); log.info("Prop " + KafkaAvroMessageEncoder.KAFKA_MESSAGE_CODER_SCHEMA_REGISTRY_CLASS + " is: " + props.getProperty(KafkaAvroMessageEncoder.KAFKA_MESSAGE_CODER_SCHEMA_REGISTRY_CLASS)); log.info("Underlying schema registry for topic: " + topicName + " is: " + registry); registry.init(props); this.registry = new CachedSchemaRegistry<Schema>(registry, props); this.latestSchema = ((Schema) registry.getLatestSchemaByTopic(topicName).getSchema()); } catch (Exception e) { throw new MessageDecoderException(e); } this.decoderFactory = DecoderFactory.get(); }
public SchemaDetails<S> getLatestSchemaByTopic(String topicName) { SchemaDetails<S> schema = cachedLatest.get(topicName); if (schema == null) { schema = registry.getLatestSchemaByTopic(topicName); cachedLatest.putIfAbsent(topicName, schema); } return schema; }
public String register(String topic, S schema) { return registry.register(topic, schema); }
private synchronized S fetchFromSchemaRegistry(String topic, String id) { try { S schema = registry.getSchemaByID(topic, id); return schema; } catch (SchemaNotFoundException e) { addFetchToFailureHistory(id); throw e; } }
if (!cache.containsKey(schema)) { try { id = client.register(topicName, record.getSchema()); cache.put(schema, id); } catch (SchemaRegistryException e) {
@Test public void testMinInterval() throws InterruptedException { EasyMock.expect(registry.getSchemaByID(EasyMock.anyString(), EasyMock.anyString())).andThrow( new SchemaNotFoundException()); EasyMock.expectLastCall().times(4); EasyMock.replay(registry); props.setProperty(GET_SCHEMA_BY_ID_MAX_RETIRES, String.valueOf(Integer.MAX_VALUE)); props.setProperty(GET_SCHEMA_BY_ID_MIN_INTERVAL_SECONDS, "2"); cachedRegistry = new CachedSchemaRegistry<Schema>(registry, props); for (int i = 0; i < 2; i++) { for (int j = 0; j < 5; j++) { try { cachedRegistry.getSchemaByID("dummyTopic", "dummyID"); } catch (SchemaNotFoundException e) { } try { cachedRegistry.getSchemaByID("dummyTopic", "dummyID2"); } catch (SchemaNotFoundException e) { } } Thread.sleep(2500); } EasyMock.verify(registry); } }
@Test public void testMaxRetries() { EasyMock.expect(registry.getSchemaByID(EasyMock.anyString(), EasyMock.anyString())).andThrow( new SchemaNotFoundException()); EasyMock.expectLastCall().times(20); EasyMock.replay(registry); props.setProperty(GET_SCHEMA_BY_ID_MAX_RETIRES, "10"); props.setProperty(GET_SCHEMA_BY_ID_MIN_INTERVAL_SECONDS, "0"); cachedRegistry = new CachedSchemaRegistry<Schema>(registry, props); for (int i = 0; i < 100; i++) { try { cachedRegistry.getSchemaByID("dummyTopic", "dummyID"); } catch (SchemaNotFoundException e) { } try { cachedRegistry.getSchemaByID("dummyTopic", "dummyID2"); } catch (SchemaNotFoundException e) { } } EasyMock.verify(registry); }
private synchronized S fetchFromSchemaRegistry(String topic, String id) { try { S schema = registry.getSchemaByID(topic, id); return schema; } catch (SchemaNotFoundException e) { addFetchToFailureHistory(id); throw e; } }
@Test public void testWithSchema() { SchemaRegistry schemaRegistry = createMock(SchemaRegistry.class); Schema schema = SchemaBuilder.record("testRecord").namespace("my.name.space") .fields().name("field").type().stringType().noDefault().endRecord(); expect(schemaRegistry.getSchemaByID("myTopic", "1751217253")).andReturn(schema); replay(schemaRegistry); KafkaAvroMessageDecoder kafkaAvroMessageDecoder = new KafkaAvroMessageDecoder(); final byte[] bytes = "whatever".getBytes(); bytes[0] = 0x0; // Magic byte KafkaAvroMessageDecoder.MessageDecoderHelper messageDecoderHelper = kafkaAvroMessageDecoder.new MessageDecoderHelper(schemaRegistry, "myTopic", bytes); KafkaAvroMessageDecoder.MessageDecoderHelper actualResult = messageDecoderHelper.invoke(); verify(schemaRegistry); assertEquals("my.name.space", actualResult.getSchema().getNamespace()); assertEquals(5, actualResult.getStart()); assertEquals(bytes, actualResult.getBuffer().array()); assertEquals(3, actualResult.getLength()); }