@Override public final O serialize(I input, SchemaMetadata schemaMetadata) throws SerDesException { ensureInitialized(); // compute schema based on input object String schema = getSchemaText(input); // register that schema and get the version try { SchemaIdVersion schemaIdVersion = schemaRegistryClient.addSchemaVersion(schemaMetadata, new SchemaVersion(schema, "Schema registered by serializer:" + this.getClass())); // write the version and given object to the output return doSerialize(input, schemaIdVersion); } catch (SchemaNotFoundException | IncompatibleSchemaException | InvalidSchemaException | SchemaBranchNotFoundException e) { throw new RegistryException(e); } }
SchemaVersion avroSchemaVersion = new SchemaVersion(avroSchemaText, streamsSchemaInfo.getSchemaVersion().getDescription()); schemaIdVersion = schemaRegistryClient.addSchemaVersion(schemaName, avroSchemaVersion); } catch (InvalidSchemaException e) {
public void runCustomSerDesApi() throws Exception { // upload jar file String serdesJarName = "/serdes-examples.jar"; InputStream serdesJarInputStream = SampleSchemaRegistryClientApp.class.getResourceAsStream(serdesJarName); if (serdesJarInputStream == null) { throw new RuntimeException("Jar " + serdesJarName + " could not be loaded"); } String fileId = schemaRegistryClient.uploadFile(serdesJarInputStream); SchemaMetadata schemaMetadata = createSchemaMetadata("serdes-device-" + System.currentTimeMillis()); SchemaIdVersion v1 = schemaRegistryClient.addSchemaVersion(schemaMetadata, new SchemaVersion(getSchema("/device.avsc"), "Initial version of the schema")); // register serializer/deserializer Long serDesId = registerSimpleSerDes(fileId); // map serializer and deserializer with schemakey // for each schema, one serializer/deserializer is sufficient unless someone want to maintain multiple implementations of serializers/deserializers String schemaName = schemaMetadata.getName(); schemaRegistryClient.mapSchemaWithSerDes(schemaName, serDesId); SnapshotSerializer<Object, byte[], SchemaMetadata> snapshotSerializer = getSnapshotSerializer(schemaMetadata); String payload = "Random text: " + new Random().nextLong(); byte[] serializedBytes = snapshotSerializer.serialize(payload, schemaMetadata); SnapshotDeserializer<byte[], Object, Integer> snapshotdeserializer = getSnapshotDeserializer(schemaMetadata); Object deserializedObject = snapshotdeserializer.deserialize(serializedBytes, null); LOG.info("Given payload and deserialized object are equal: " + payload.equals(deserializedObject)); }
private SchemaIdVersion addSchemaVersion(String schemaBranchName, SchemaMetadata schemaMetadata, String pathToSchema) throws IOException, SchemaNotFoundException, InvalidSchemaException, IncompatibleSchemaException { String schema = AvroSchemaRegistryClientUtil.getSchema(pathToSchema); SchemaIdVersion schemaIdVersion = schemaRegistryClient.addSchemaVersion(schemaBranchName, schemaMetadata, new SchemaVersion(schema, "schema version description")); return schemaIdVersion; }
@Test(expected = SchemaNotFoundException.class) public void testAddVersionToNonExistingSchema() throws SchemaNotFoundException, IncompatibleSchemaException, InvalidSchemaException, SchemaBranchNotFoundException { schemaRegistry.addSchemaVersion(INVALID_SCHEMA_METADATA_KEY, new SchemaVersion("foo", "dummy")); }
@Override public SchemaIdVersion addSchemaVersion(SchemaMetadata schemaMetadata, SchemaVersion schemaVersion) throws InvalidSchemaException, IncompatibleSchemaException, SchemaNotFoundException, SchemaBranchNotFoundException { try { return schemaRegistry.addSchemaVersion(schemaMetadata, new SchemaVersion(schemaVersion.getSchemaText(), schemaMetadata.getDescription())); } catch (UnsupportedSchemaTypeException e) { throw new RuntimeException(e); } }
@Override public SchemaIdVersion addSchemaVersion(String schemaName, SchemaVersion schemaVersion) throws InvalidSchemaException, IncompatibleSchemaException, SchemaNotFoundException, SchemaBranchNotFoundException { try { return schemaRegistry.addSchemaVersion(schemaName, new SchemaVersion(schemaVersion.getSchemaText(), schemaVersion.getDescription())); } catch (UnsupportedSchemaTypeException e) { throw new RuntimeException(e); } }
@Test(expected = IncompatibleSchemaException.class) public void testIncompatibleSchemas() throws Exception { String schema = AvroSchemaRegistryClientUtil.getSchema("/device.avsc"); String incompatSchema = AvroSchemaRegistryClientUtil.getSchema("/device-incompat.avsc"); SchemaMetadata schemaMetadata = createSchemaMetadata(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BACKWARD); // registering a new schema SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, new SchemaVersion(schema, "Initial version of the schema")); // adding a new version of the schema SchemaVersion incompatSchemaInfo = new SchemaVersion(incompatSchema, "second version"); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, incompatSchemaInfo); }
@Test(expected = InvalidSchemaException.class) public void testInvalidSchema() throws Exception { String schema = "--- random invalid schema ---" + new Date(); SchemaMetadata schemaMetadataInfo = createSchemaInfo(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BACKWARD); // registering a new schema Integer v1 = schemaRegistry.addSchemaVersion(schemaMetadataInfo, new SchemaVersion(schema, "Initial version of the schema")) .getVersion(); }
@Test(expected = InvalidSchemaException.class) public void testInvalidSchema() throws Exception { String schema = "--- invalid schema ---"; SchemaMetadata schemaMetadata = createSchemaMetadata(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BACKWARD); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, new SchemaVersion(schema, "Initial version of the schema")); }
@Test(expected = InvalidSchemaException.class) public void testSchemaTextAsNull() throws Exception { SchemaMetadata schemaMetadataInfo = createSchemaInfo(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BACKWARD); // registering a new schema Integer v1 = schemaRegistry.addSchemaVersion(schemaMetadataInfo, new SchemaVersion(null, "Initial version of the schema")) .getVersion(); }
@Test(expected = InvalidSchemaException.class) public void testSchemaTextAsEmpty() throws Exception { SchemaMetadata schemaMetadataInfo = createSchemaInfo(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BACKWARD); // registering a new schema Integer v1 = schemaRegistry.addSchemaVersion(schemaMetadataInfo, new SchemaVersion("", "Initial version of the schema")) .getVersion(); }
@Test(expected = IncompatibleSchemaException.class) public void testIncompatibleSchemas() throws Exception { String schema = getSchema("/device.avsc"); String incompatSchema = getSchema("/device-incompat.avsc"); SchemaMetadata schemaMetadata = createSchemaInfo(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BACKWARD); // registering a new schema Integer v1 = schemaRegistry.addSchemaVersion(schemaMetadata, new SchemaVersion(schema, "Initial version of the schema")) .getVersion(); // adding a new version of the schema Integer v2 = schemaRegistry.addSchemaVersion(schemaMetadata, new SchemaVersion(incompatSchema, "second version")) .getVersion(); }
@Test public void testSerializerOps() throws Exception { String fileId = uploadFile(); SchemaMetadata schemaMetadata = createSchemaMetadata(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BOTH); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/device.avsc"), "Initial version of the schema")); SerDesPair serDesPair = createSerDesInfo(fileId); Long serDesId = SCHEMA_REGISTRY_CLIENT.addSerDes(serDesPair); Assert.assertNotNull("Returned serDesId can not be null", serDesId); String schemaName = schemaMetadata.getName(); SCHEMA_REGISTRY_CLIENT.mapSchemaWithSerDes(schemaName, serDesId); Collection<SerDesInfo> serializers = SCHEMA_REGISTRY_CLIENT.getSerDes(schemaName); Assert.assertTrue(serializers.stream() .map(x -> x.getSerDesPair()) .collect(Collectors.toList()) .contains(serDesPair)); }
private void _testAvroSerDesGenericObj(Byte protocolId) throws IOException, InvalidSchemaException, IncompatibleSchemaException, SchemaNotFoundException, SchemaBranchNotFoundException { Map<String, Object> config = Maps.newHashMap(); config.putAll(SCHEMA_REGISTRY_CLIENT_CONF); config.put(SERDES_PROTOCOL_VERSION, protocolId); AvroSnapshotSerializer avroSnapshotSerializer = new AvroSnapshotSerializer(); avroSnapshotSerializer.init(config); AvroSnapshotDeserializer avroSnapshotDeserializer = new AvroSnapshotDeserializer(); avroSnapshotDeserializer.init(config); String deviceSchema = AvroSchemaRegistryClientUtil.getSchema("/device.avsc"); SchemaMetadata schemaMetadata = createSchemaMetadata(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BOTH); SchemaIdVersion v1 = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, new SchemaVersion(deviceSchema, "Initial version of the schema")); Assert.assertNotNull(v1); Object deviceObject = AvroSchemaRegistryClientUtil.createGenericRecordForDevice(); byte[] serializedData = avroSnapshotSerializer.serialize(deviceObject, schemaMetadata); Object deserializedObj = avroSnapshotDeserializer.deserialize(new ByteArrayInputStream(serializedData), null); Assert.assertEquals(deviceObject, deserializedObj); }
private SchemaVersionKey addAndDeleteSchemaVersion(String schemaName) throws InvalidSchemaException, IncompatibleSchemaException, SchemaNotFoundException, IOException, SchemaBranchNotFoundException, SchemaLifecycleException { SchemaMetadata schemaMetadata = createSchemaMetadata(schemaName, SchemaCompatibility.BOTH); SchemaIdVersion schemaIdVersion = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, new SchemaVersion(AvroSchemaRegistryClientUtil .getSchema("/device.avsc"), "Initial version of the schema")); SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaMetadata.getName(), schemaIdVersion.getVersion()); SCHEMA_REGISTRY_CLIENT.deleteSchemaVersion(schemaVersionKey); return schemaVersionKey; }
@Test public void testGetSchemaVersionFromStates() throws IOException, SchemaNotFoundException, InvalidSchemaException, IncompatibleSchemaException, SchemaLifecycleException, SchemaBranchAlreadyExistsException { SchemaMetadata schemaMetadata = createSchemaMetadata(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.NONE); String schemaName = schemaMetadata.getName(); Long id = SCHEMA_REGISTRY_CLIENT.registerSchemaMetadata(schemaMetadata); SchemaIdVersion v1 = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-1.avsc"), "Initial version of the schema")); SchemaIdVersion v2 = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-2.avsc"), "Second version of the schema")); SchemaIdVersion v3 = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-3.avsc"), "Third version of the schema, removes name field")); SchemaBranch schemaBranch = SCHEMA_REGISTRY_CLIENT.createSchemaBranch(v3.getSchemaVersionId(), new SchemaBranch("Branch-1", schemaName)); SchemaIdVersion v4 = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaBranch.getName(), schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-4.avsc"), "Forth version of the schema, adds back name field, but different type")); SCHEMA_REGISTRY_CLIENT.startSchemaVersionReview(v4.getSchemaVersionId()); SCHEMA_REGISTRY_CLIENT.transitionState(v4.getSchemaVersionId(), SchemaVersionLifecycleStates.REVIEWED.getId(), null); SCHEMA_REGISTRY_CLIENT.archiveSchemaVersion(v2.getSchemaVersionId()); SCHEMA_REGISTRY_CLIENT.archiveSchemaVersion(v3.getSchemaVersionId()); Assert.assertEquals( transformToSchemaIdVersions(SCHEMA_REGISTRY_CLIENT.getAllVersions(SchemaBranch.MASTER_BRANCH, schemaName, Collections.singletonList(SchemaVersionLifecycleStates.ENABLED.getId()))), new HashSet<>(Arrays.asList(v1))); Assert.assertEquals( transformToSchemaIdVersions(SCHEMA_REGISTRY_CLIENT.getAllVersions(SchemaBranch.MASTER_BRANCH, schemaName, Collections.singletonList(SchemaVersionLifecycleStates.ARCHIVED.getId()))), new HashSet<>(Arrays.asList(v2,v3))); Assert.assertEquals( transformToSchemaIdVersions(SCHEMA_REGISTRY_CLIENT.getAllVersions(schemaBranch.getName(), schemaName, Collections.singletonList(SchemaVersionLifecycleStates.REVIEWED.getId()))), new HashSet<>(Arrays.asList(v4))); SCHEMA_REGISTRY_CLIENT.disableSchemaVersion(v1.getSchemaVersionId()); Assert.assertEquals( transformToSchemaIdVersions(SCHEMA_REGISTRY_CLIENT.getAllVersions(SchemaBranch.MASTER_BRANCH, schemaName, Arrays.asList(SchemaVersionLifecycleStates.ARCHIVED.getId(), SchemaVersionLifecycleStates.DISABLED.getId()))), new HashSet<>(Arrays.asList(v1,v2,v3))); }
@Test public void testValidationLevels() throws Exception { SchemaMetadata schemaMetadata = createSchemaMetadata(TEST_NAME_RULE.getMethodName(), SchemaCompatibility.BOTH); String schemaName = schemaMetadata.getName(); Long id = SCHEMA_REGISTRY_CLIENT.registerSchemaMetadata(schemaMetadata); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-1.avsc"), "Initial version of the schema")); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-2.avsc"), "Second version of the schema")); SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-3.avsc"), "Third version of the schema, removes name field")); try { SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-4.avsc"), "Forth version of the schema, adds back name field, but different type")); Assert.fail("Should throw IncompatibleSchemaException as check against all schema's would find name field is not compatible with v1 and v2"); } catch (IncompatibleSchemaException ise) { //expected } SchemaMetadata currentSchemaMetadata = SCHEMA_REGISTRY_CLIENT.getSchemaMetadataInfo(schemaName) .getSchemaMetadata(); SchemaMetadata schemaMetadataToUpdateTo = new SchemaMetadata.Builder(currentSchemaMetadata).validationLevel(SchemaValidationLevel.LATEST) .build(); SchemaMetadataInfo updatedSchemaMetadata = SCHEMA_REGISTRY_CLIENT.updateSchemaMetadata(schemaName, schemaMetadataToUpdateTo); Assert.assertEquals(SchemaValidationLevel.LATEST, updatedSchemaMetadata.getSchemaMetadata() .getValidationLevel()); try { SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaName, new SchemaVersion(AvroSchemaRegistryClientUtil.getSchema("/schema-4.avsc"), "Forth version of the schema, adds back name field, but different type")); } catch (IncompatibleSchemaException ise) { Assert.fail("Should not throw IncompatibleSchemaException as check against only latest schema as such should ignore v1 and v2"); } }
@Test public void testSanity() throws Exception { SchemaRegistryTestServerClientWrapper schemaRegistryTestServerClientWrapper = new SchemaRegistryTestServerClientWrapper(SCHEMA_REGISTRY_TEST_CONFIGURATION); schemaRegistryTestServerClientWrapper.startTestServer(); SchemaRegistryClient schemaRegistryClient = schemaRegistryTestServerClientWrapper.getClient(); // registering schema metadata SchemaMetadata schemaMetadata = new SchemaMetadata.Builder("foo").type("avro").build(); Long schemaId = schemaRegistryClient.registerSchemaMetadata(schemaMetadata); Assert.assertNotNull(schemaId); // registering a new schema String schemaName = schemaMetadata.getName(); String schema1 = IOUtils.toString(LocalRegistryServerTest.class.getResourceAsStream("/schema-1.avsc"), "UTF-8"); SchemaIdVersion v1 = schemaRegistryClient.addSchemaVersion(schemaName, new SchemaVersion(schema1, "Initial version of the schema")); schemaRegistryTestServerClientWrapper.stopTestServer(); } }
@Test (expected = InvalidSchemaException.class) public void addInvalidSchemaToBranch() throws IOException, SchemaBranchNotFoundException, InvalidSchemaException, SchemaNotFoundException, IncompatibleSchemaException, SchemaBranchAlreadyExistsException { SchemaMetadata schemaMetadata = addSchemaMetadata(testNameRule.getMethodName(), SchemaCompatibility.BACKWARD); SchemaIdVersion masterSchemaIdVersion = addSchemaVersion(SchemaBranch.MASTER_BRANCH, schemaMetadata, "/device.avsc"); SchemaBranch schemaBranch1 = addSchemaBranch("BRANCH1", schemaMetadata, masterSchemaIdVersion.getSchemaVersionId()); String schema2 = "--- invalid schema ---"; schemaRegistryClient.addSchemaVersion(schemaBranch1.getName(), schemaMetadata.getName(), new SchemaVersion(schema2, "second version")); }