@Override synchronized public K register(String name, S schema) throws IOException, SchemaRegistryException { Map<S, K> schemaIdMap; if (_namedSchemaCache.containsKey(name)) { schemaIdMap = _namedSchemaCache.get(name); } else { // we really care about reference equality to de-dup using cache // when it comes to registering schemas, so use an IdentityHashMap here schemaIdMap = new IdentityHashMap<>(); _namedSchemaCache.put(name, schemaIdMap); } if (schemaIdMap.containsKey(schema)) { return schemaIdMap.get(schema); } else { // check if schemaIdMap is getting too full Preconditions.checkState(schemaIdMap.size() < _maxSchemaReferences, "Too many schema objects for " + name +". Cache is overfull."); } K id = _kafkaSchemaRegistry.register(name, schema); schemaIdMap.put(schema, id); _idBasedCache.put(id, schema); return id; }
public byte[] serialize(String topic, GenericRecord data) throws SerializationException { Schema schema = data.getSchema(); MD5Digest schemaId = null; try { schemaId = schemaRegistry.register(topic, schema); ByteArrayOutputStream out = new ByteArrayOutputStream(); // MAGIC_BYTE | schemaId-bytes | avro_payload out.write(LiAvroSerDeHelper.MAGIC_BYTE); out.write(schemaId.asBytes()); BinaryEncoder encoder = encoderFactory.directBinaryEncoder(out, null); DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema); writer.write(data, encoder); encoder.flush(); byte[] bytes = out.toByteArray(); out.close(); return bytes; } catch (IOException | SchemaRegistryException e) { throw new SerializationException(e); } }
@Test public void testRegisterSchemaCaching() throws IOException, SchemaRegistryException { KafkaSchemaRegistry<Integer, String> baseRegistry = mock(KafkaSchemaRegistry.class); String name = "test"; String schema1 = new String("schema"); Integer id1 = 1; CachingKafkaSchemaRegistry<Integer, String> cachingReg = new CachingKafkaSchemaRegistry<>(baseRegistry, 2); when(baseRegistry.register(name, schema1)).thenReturn(id1); Assert.assertEquals(cachingReg.register(name, schema1), id1); Integer id2 = 2; when(baseRegistry.register(name, schema1)).thenReturn(id2); // Test that we get back the original id Assert.assertEquals(cachingReg.register(name, schema1), id1); // Ensure that we only called baseRegistry.register once. verify(baseRegistry, times(1)).register(anyString(), anyString()); }
@Test public void testRegisterShouldCacheIds() throws IOException, SchemaRegistryException { KafkaSchemaRegistry<Integer, String> baseRegistry = mock(KafkaSchemaRegistry.class); CachingKafkaSchemaRegistry<Integer, String> cachingReg = new CachingKafkaSchemaRegistry<>(baseRegistry, 2); String name = "test"; String schema1 = new String("schema"); Integer id1 = 1; // first register name, schema1, get back id1 when(baseRegistry.register(name, schema1)).thenReturn(id1); Assert.assertEquals(cachingReg.register(name, schema1), id1); // getById should hit the cache and return id1 when(baseRegistry.getById(id1)).thenReturn(new String("schema2")); Assert.assertEquals(cachingReg.getById(id1), schema1); verify(baseRegistry, times(0)).getById(anyInt()); }
@Test public void testMaxReferences() throws IOException, SchemaRegistryException { KafkaSchemaRegistry<Integer, String> baseRegistry = mock(KafkaSchemaRegistry.class); String name = "test"; String schema1 = new String("schema"); String schema2 = new String("schema"); String schema3 = new String("schema"); Integer id1 = 1; Integer id2 = 2; Integer id3 = 3; CachingKafkaSchemaRegistry<Integer, String> cachingReg = new CachingKafkaSchemaRegistry<>(baseRegistry, 2); when(baseRegistry.register(name, schema1)).thenReturn(id1); Assert.assertEquals(cachingReg.register(name, schema1), id1); when(baseRegistry.register(name, schema2)).thenReturn(id2); Assert.assertEquals(cachingReg.register(name, schema2), id2); when(baseRegistry.register(name, schema3)).thenReturn(id3); try { cachingReg.register(name, schema3); Assert.fail("Should have thrown an exception"); } catch (Exception e) { log.info(e.getMessage()); } }
@Override synchronized public K register(String name, S schema) throws IOException, SchemaRegistryException { Map<S, K> schemaIdMap; if (_namedSchemaCache.containsKey(name)) { schemaIdMap = _namedSchemaCache.get(name); } else { // we really care about reference equality to de-dup using cache // when it comes to registering schemas, so use an IdentityHashMap here schemaIdMap = new IdentityHashMap<>(); _namedSchemaCache.put(name, schemaIdMap); } if (schemaIdMap.containsKey(schema)) { return schemaIdMap.get(schema); } else { // check if schemaIdMap is getting too full Preconditions.checkState(schemaIdMap.size() < _maxSchemaReferences, "Too many schema objects for " + name +". Cache is overfull."); } K id = _kafkaSchemaRegistry.register(name, schema); schemaIdMap.put(schema, id); _idBasedCache.put(id, schema); return id; }
public byte[] serialize(String topic, GenericRecord data) throws SerializationException { Schema schema = data.getSchema(); MD5Digest schemaId = null; try { schemaId = schemaRegistry.register(topic, schema); ByteArrayOutputStream out = new ByteArrayOutputStream(); // MAGIC_BYTE | schemaId-bytes | avro_payload out.write(LiAvroSerDeHelper.MAGIC_BYTE); out.write(schemaId.asBytes()); BinaryEncoder encoder = encoderFactory.directBinaryEncoder(out, null); DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema); writer.write(data, encoder); encoder.flush(); byte[] bytes = out.toByteArray(); out.close(); return bytes; } catch (IOException | SchemaRegistryException e) { throw new SerializationException(e); } }