final SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaName.get(), schemaVersion.getAsInt()); versionInfo = getSchemaVersionInfo(client, schemaVersionKey); } else {
schemaName = metadata.getName(); final SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaName, version.getAsInt()); versionInfo = getSchemaVersionInfo(client, schemaVersionKey); if (versionInfo == null) {
private List<SchemaVersionKey> getIncludedSchemaVersions(String schemaText) throws InvalidSchemaException { JsonNode jsonNode = null; try { jsonNode = new ObjectMapper().readTree(schemaText); } catch (IOException e) { throw new InvalidSchemaException(e); } JsonNode includeSchemaNodes = jsonNode.get("includeSchemas"); List<SchemaVersionKey> includedSchemaVersions = new ArrayList<>(); if (includeSchemaNodes != null) { if (!includeSchemaNodes.isArray()) { throw new InvalidSchemaException("includeSchemas should be an array of strings"); } for (JsonNode includeSchema : includeSchemaNodes) { String name = includeSchema.get("name").asText(); JsonNode versionNode = includeSchema.get("version"); int version = versionNode != null ? versionNode.asInt() : SchemaVersionKey.LATEST_VERSION; includedSchemaVersions.add(new SchemaVersionKey(name, version)); } } return includedSchemaVersions; } }
private SchemaVersionKey getSchemaKey(Long schemaId) { SchemaVersionKey schemaVersionKey = null; List<QueryParam> queryParams = Collections.singletonList(new QueryParam(SchemaVersionStorable.ID, schemaId.toString())); Collection<SchemaVersionStorable> versionedSchemas = storageManager.find(SchemaVersionStorable.NAME_SPACE, queryParams); if (versionedSchemas != null && !versionedSchemas.isEmpty()) { SchemaVersionStorable storable = versionedSchemas.iterator().next(); schemaVersionKey = new SchemaVersionKey(storable.getName(), storable.getVersion()); } return schemaVersionKey; }
private List<SchemaVersionKey> getIncludedSchemaVersions(String schemaText) throws InvalidSchemaException { JsonNode jsonNode = null; try { jsonNode = new ObjectMapper().readTree(schemaText); } catch (IOException e) { throw new InvalidSchemaException(e); } JsonNode includeSchemaNodes = jsonNode.get("includeSchemas"); List<SchemaVersionKey> includedSchemaVersions = new ArrayList<>(); if (includeSchemaNodes != null) { if (!includeSchemaNodes.isArray()) { throw new InvalidSchemaException("includeSchemas should be an array of strings"); } for (JsonNode includeSchema : includeSchemaNodes) { String name = includeSchema.get("name").asText(); JsonNode versionNode = includeSchema.get("version"); int version = versionNode != null ? versionNode.asInt() : SchemaVersionKey.LATEST_VERSION; includedSchemaVersions.add(new SchemaVersionKey(name, version)); } } return includedSchemaVersions; } }
@GET @Path("/schemas/{name}/versions/{version}") @ApiOperation(value = "Get a version of the schema identified by the schema name", response = SchemaVersionInfo.class, tags = OPERATION_GROUP_SCHEMA) @Timed @UnitOfWork public Response getSchemaVersion(@ApiParam(value = "Schema name", required = true) @PathParam("name") String schemaMetadata, @ApiParam(value = "version of the schema", required = true) @PathParam("version") Integer versionNumber) { SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaMetadata, versionNumber); Response response; try { SchemaVersionInfo schemaVersionInfo = schemaRegistry.getSchemaVersionInfo(schemaVersionKey); response = WSUtils.respondEntity(schemaVersionInfo, Response.Status.OK); } catch (SchemaNotFoundException e) { LOG.info("No schemas found with schemaVersionKey: [{}]", schemaVersionKey); response = WSUtils.respond(Response.Status.NOT_FOUND, CatalogResponse.ResponseMessage.ENTITY_NOT_FOUND, schemaVersionKey.toString()); } catch (Exception ex) { LOG.error("Encountered error while getting all schema versions for schemakey [{}]", schemaMetadata, ex); response = WSUtils.respond(Response.Status.INTERNAL_SERVER_ERROR, CatalogResponse.ResponseMessage.EXCEPTION, ex.getMessage()); } return response; }
@GET @Path("/schemas/{name}/versions/{version}") @ApiOperation(value = "Get a version of the schema identified by the schema name", response = SchemaVersionInfo.class, tags = OPERATION_GROUP_SCHEMA) @Timed @UnitOfWork public Response getSchemaVersion(@ApiParam(value = "Schema name", required = true) @PathParam("name") String schemaMetadata, @ApiParam(value = "version of the schema", required = true) @PathParam("version") Integer versionNumber) { SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaMetadata, versionNumber); Response response; try { SchemaVersionInfo schemaVersionInfo = schemaRegistry.getSchemaVersionInfo(schemaVersionKey); response = WSUtils.respondEntity(schemaVersionInfo, Response.Status.OK); } catch (SchemaNotFoundException e) { LOG.info("No schemas found with schemaVersionKey: [{}]", schemaVersionKey); response = WSUtils.respond(Response.Status.NOT_FOUND, CatalogResponse.ResponseMessage.ENTITY_NOT_FOUND, schemaVersionKey.toString()); } catch (Exception ex) { LOG.error("Encountered error while getting all schema versions for schemakey [{}]", schemaMetadata, ex); response = WSUtils.respond(Response.Status.INTERNAL_SERVER_ERROR, CatalogResponse.ResponseMessage.EXCEPTION, ex.getMessage()); } return response; }
@GET @Path("/{schemaName}/versions/{version}") @Timed @Produces(MediaType.APPLICATION_JSON) public Response getSchemaForVersion(@PathParam("schemaName") String schemaName, @PathParam("version") String version, @Context SecurityContext securityContext) { try { LOG.info("Get schema:version [{}:{}]", schemaName, version); SchemaVersionInfo schemaVersionInfo = schemaRegistryClient.getSchemaVersionInfo(new SchemaVersionKey(schemaName, Integer.parseInt(version))); if (schemaVersionInfo != null) { LOG.debug("Received schema version from schema registry: [{}]", schemaVersionInfo); String schema = schemaVersionInfo.getSchemaText(); if (schema != null && !schema.isEmpty()) { schema = AvroStreamlineSchemaConverter.convertAvroSchemaToStreamlineSchema(schema); } LOG.debug("Converted schema: [{}]", schema); return WSUtils.respondEntity(schema, OK); } else { throw new SchemaNotFoundException("Version " + version + " of schema " + schemaName + " not found "); } } catch (SchemaNotFoundException e) { LOG.error("Schema not found: [{}]", schemaName, e); throw EntityNotFoundException.byName(schemaName); } catch (Exception e) { throw new RuntimeException(e); } }
@DELETE @Path("/schemas/{name}/versions/{version}") @ApiOperation(value = "Delete a schema version given its schema name and version id", tags = OPERATION_GROUP_SCHEMA) @UnitOfWork public Response deleteSchemaVersion(@ApiParam(value = "Schema name", required = true) @PathParam("name") String schemaName, @ApiParam(value = "version of the schema", required = true) @PathParam("version") Integer versionNumber, @Context UriInfo uriInfo) { SchemaVersionKey schemaVersionKey = null; try { schemaVersionKey = new SchemaVersionKey(schemaName, versionNumber); schemaRegistry.deleteSchemaVersion(schemaVersionKey); return WSUtils.respond(Response.Status.OK); } catch (SchemaNotFoundException e) { LOG.error("No schemaVersion found with name: [{}], version : [{}]", schemaName, versionNumber); return WSUtils.respond(Response.Status.NOT_FOUND, CatalogResponse.ResponseMessage.ENTITY_NOT_FOUND, schemaVersionKey.toString()); } catch (SchemaLifecycleException e) { LOG.error("Failed to delete schema name: [{}], version : [{}]", schemaName, versionNumber, e); return WSUtils.respond(Response.Status.BAD_REQUEST, CatalogResponse.ResponseMessage.BAD_REQUEST_WITH_MESSAGE, e.getMessage()); } catch (Exception ex) { LOG.error("Encountered error while deleting schemaVersion with name: [{}], version : [{}]", schemaName, versionNumber, ex); return WSUtils.respond(Response.Status.INTERNAL_SERVER_ERROR, CatalogResponse.ResponseMessage.EXCEPTION, ex.getMessage()); } }
@DELETE @Path("/schemas/{name}/versions/{version}") @ApiOperation(value = "Delete a schema version given its schema name and version id", tags = OPERATION_GROUP_SCHEMA) @UnitOfWork public Response deleteSchemaVersion(@ApiParam(value = "Schema name", required = true) @PathParam("name") String schemaName, @ApiParam(value = "version of the schema", required = true) @PathParam("version") Integer versionNumber, @Context UriInfo uriInfo) { SchemaVersionKey schemaVersionKey = null; try { schemaVersionKey = new SchemaVersionKey(schemaName, versionNumber); schemaRegistry.deleteSchemaVersion(schemaVersionKey); return WSUtils.respond(Response.Status.OK); } catch (SchemaNotFoundException e) { LOG.error("No schemaVersion found with name: [{}], version : [{}]", schemaName, versionNumber); return WSUtils.respond(Response.Status.NOT_FOUND, CatalogResponse.ResponseMessage.ENTITY_NOT_FOUND, schemaVersionKey.toString()); } catch (SchemaLifecycleException e) { LOG.error("Failed to delete schema name: [{}], version : [{}]", schemaName, versionNumber, e); return WSUtils.respond(Response.Status.BAD_REQUEST, CatalogResponse.ResponseMessage.BAD_REQUEST_WITH_MESSAGE, e.getMessage()); } catch (Exception ex) { LOG.error("Encountered error while deleting schemaVersion with name: [{}], version : [{}]", schemaName, versionNumber, ex); return WSUtils.respond(Response.Status.INTERNAL_SERVER_ERROR, CatalogResponse.ResponseMessage.EXCEPTION, ex.getMessage()); } }
private void updateCacheInvalidationEntries(SchemaVersionInfo schemaVersionInfo) { // need to support this as SchemaIdVersion supports multiple ways to construct for backward compatible APIs // this would have been simple without that. SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaVersionInfo.getName(), schemaVersionInfo.getVersion()); SchemaIdVersion key1 = new SchemaIdVersion(schemaVersionInfo.getId()); idWithNameVersion.putIfAbsent(key1, schemaVersionKey); Long schemaMetadataId = schemaVersionInfo.getSchemaMetadataId(); // schemaMetadataId can be null from earlier registry instances. if (schemaMetadataId != null) { SchemaIdVersion key2 = new SchemaIdVersion(schemaMetadataId, schemaVersionInfo.getVersion()); nameVersionWithIds.putIfAbsent(schemaVersionKey, Lists.newArrayList(key1, key2)); idWithNameVersion.putIfAbsent(key2, schemaVersionKey); } else { nameVersionWithIds.putIfAbsent(schemaVersionKey, Collections.singletonList(key1)); } }
private void updateCacheInvalidationEntries(SchemaVersionInfo schemaVersionInfo) { // need to support this as SchemaIdVersion supports multiple ways to construct for backward compatible APIs // this would have been simple without that. SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaVersionInfo.getName(), schemaVersionInfo.getVersion()); SchemaIdVersion key1 = new SchemaIdVersion(schemaVersionInfo.getId()); idWithNameVersion.putIfAbsent(key1, schemaVersionKey); Long schemaMetadataId = schemaVersionInfo.getSchemaMetadataId(); // schemaMetadataId can be null from earlier registry instances. if (schemaMetadataId != null) { SchemaIdVersion key2 = new SchemaIdVersion(schemaMetadataId, schemaVersionInfo.getVersion()); nameVersionWithIds.putIfAbsent(schemaVersionKey, Lists.newArrayList(key1, key2)); idWithNameVersion.putIfAbsent(key2, schemaVersionKey); } else { nameVersionWithIds.putIfAbsent(schemaVersionKey, Collections.singletonList(key1)); } }
/** * Builds the deserialized object from the given {@code payloadInputStream} and applying writer and reader schemas * from the respective given versions. * * @param protocolId protocol id * @param payloadInputStream payload * @param schemaMetadata metadata about schema * @param writerSchemaVersion schema version of the writer * @param readerSchemaVersion schema version to be applied for reading or projection * @return the deserialized object * @throws SerDesException when any ser/des error occurs */ protected Object buildDeserializedObject(byte protocolId, InputStream payloadInputStream, SchemaMetadata schemaMetadata, Integer writerSchemaVersion, Integer readerSchemaVersion) throws SerDesException { String schemaName = schemaMetadata.getName(); SchemaVersionKey writerSchemaVersionKey = new SchemaVersionKey(schemaName, writerSchemaVersion); LOG.debug("SchemaKey: [{}] for the received payload", writerSchemaVersionKey); Schema writerSchema = getSchema(writerSchemaVersionKey); if (writerSchema == null) { throw new RegistryException("No schema exists with metadata-key: " + schemaMetadata + " and writerSchemaVersion: " + writerSchemaVersion); } Schema readerSchema = readerSchemaVersion != null ? getSchema(new SchemaVersionKey(schemaName, readerSchemaVersion)) : null; return deserializePayloadForProtocol(protocolId, payloadInputStream, writerSchema, readerSchema); }
private SchemaVersionInfo doGetSchemaVersionInfo(SchemaIdVersion schemaIdVersion) throws SchemaNotFoundException { if (schemaIdVersion.getSchemaVersionId() != null) { LOG.info("Getting schema version from target registry for [{}]", schemaIdVersion.getSchemaVersionId()); return getEntity(currentSchemaRegistryTargets() .schemaVersionsByIdTarget .path(schemaIdVersion.getSchemaVersionId().toString()), SchemaVersionInfo.class); } else if (schemaIdVersion.getSchemaMetadataId() != null) { SchemaMetadataInfo schemaMetadataInfo = getSchemaMetadataInfo(schemaIdVersion.getSchemaMetadataId()); SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaMetadataInfo.getSchemaMetadata() .getName(), schemaIdVersion.getVersion()); LOG.info("Getting schema version from target registry for key [{}]", schemaVersionKey); return doGetSchemaVersionInfo(schemaVersionKey); } throw new IllegalArgumentException("Given argument not valid: " + schemaIdVersion); }
@Override public byte[] serialize(String topic, StreamlineEvent streamlineEvent) { SchemaMetadata schemaMetadata = getSchemaKey(topic, false); SchemaVersionInfo schemaVersionInfo; try { schemaMetadata = schemaRegistryClient.getSchemaMetadataInfo(schemaMetadata.getName()).getSchemaMetadata(); if (writerSchemaVersion != null) { schemaVersionInfo = schemaRegistryClient.getSchemaVersionInfo(new SchemaVersionKey(schemaMetadata.getName(), writerSchemaVersion)); } else { schemaVersionInfo = schemaRegistryClient.getLatestSchemaVersionInfo(schemaMetadata.getName()); } } catch (SchemaNotFoundException e) { LOG.error("Exception occured while getting SchemaVersionInfo for " + schemaMetadata, e); throw new RuntimeException(e); } if (streamlineEvent == null || streamlineEvent.isEmpty()) { return null; } else { return avroSnapshotSerializer.serialize(getAvroRecord(streamlineEvent, new Schema.Parser().parse(schemaVersionInfo.getSchemaText())), schemaMetadata); } }
private SchemaIdVersion handleSchemaIdVersionResponse(SchemaMetadataInfo schemaMetadataInfo, Response response) throws IncompatibleSchemaException, InvalidSchemaException { int status = response.getStatus(); String msg = response.readEntity(String.class); if (status == Response.Status.BAD_REQUEST.getStatusCode() || status == Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()) { CatalogResponse catalogResponse = readCatalogResponse(msg); if (CatalogResponse.ResponseMessage.INCOMPATIBLE_SCHEMA.getCode() == catalogResponse.getResponseCode()) { throw new IncompatibleSchemaException(catalogResponse.getResponseMessage()); } else if (CatalogResponse.ResponseMessage.INVALID_SCHEMA.getCode() == catalogResponse.getResponseCode()) { throw new InvalidSchemaException(catalogResponse.getResponseMessage()); } else { throw new RuntimeException(catalogResponse.getResponseMessage()); } } Integer version = readEntity(msg, Integer.class); SchemaVersionInfo schemaVersionInfo = doGetSchemaVersionInfo(new SchemaVersionKey(schemaMetadataInfo.getSchemaMetadata() .getName(), version)); return new SchemaIdVersion(schemaMetadataInfo.getId(), version, schemaVersionInfo.getId()); }
SchemaVersionInfo schemaVersionInfo = schemaRegistryClient.getSchemaVersionInfo(new SchemaVersionKey(schemaName, v2.getVersion())); LOG.info("Received schema version info [{}] for schema metadata [{}]", schemaVersionInfo, schemaMetadata);
private SchemaVersionKey addAndDeleteSchemaVersion(String schemaName) throws InvalidSchemaException, IncompatibleSchemaException, SchemaNotFoundException, IOException, SchemaBranchNotFoundException, SchemaLifecycleException { SchemaMetadata schemaMetadata = createSchemaMetadata(schemaName, SchemaCompatibility.BOTH); SchemaIdVersion schemaIdVersion = SCHEMA_REGISTRY_CLIENT.addSchemaVersion(schemaMetadata, new SchemaVersion(AvroSchemaRegistryClientUtil .getSchema("/device.avsc"), "Initial version of the schema")); SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaMetadata.getName(), schemaIdVersion.getVersion()); SCHEMA_REGISTRY_CLIENT.deleteSchemaVersion(schemaVersionKey); return schemaVersionKey; }
@Test public void testHASanity() throws Exception { SchemaRegistryTestServerClientWrapper followerServer = followerSchemaRegistryServer(); SchemaRegistryClient schemaRegistryClient = followerServer.getClient(); // registering schema metadata on follower, this should have been redirected to leader. String schemaName = "foo"; SchemaMetadata schemaMetadata = new SchemaMetadata.Builder(schemaName).type("avro").build(); Long schemaId = schemaRegistryClient.registerSchemaMetadata(schemaMetadata); Assert.assertNotNull(schemaId); // registering a new schema on follower, this should have been redirected to leader. String schema1 = IOUtils.toString(LocalRegistryServerTest.class.getResourceAsStream("/schema-1.avsc"), "UTF-8"); SchemaIdVersion v1 = schemaRegistryClient.addSchemaVersion(schemaName, new SchemaVersion(schema1, "Initial version of the schema")); // retrieve schema on leader as the schema data is stored in memory in leader. this data does not exist on // followers as the storage is inmemory. SchemaRegistryTestServerClientWrapper leaderServer = leaderSchemaRegistryServer(); int leaderPort = leaderServer.getLocalPort(); SchemaRegistryClient leaderClient = leaderServer.getClient(); SchemaMetadataInfo schemaMetadataInfo = leaderClient.getSchemaMetadataInfo(schemaName); Assert.assertEquals(schemaMetadata, schemaMetadataInfo.getSchemaMetadata()); // stop the leader server leaderServer.stopTestServer(); // get the new leader server and run operations. SchemaRegistryTestServerClientWrapper newLeaderServer = leaderSchemaRegistryServer(); Assert.assertNotEquals(leaderPort, newLeaderServer.getLocalPort()); leaderClient = newLeaderServer.getClient(); String receivedSchema = leaderClient.getSchemaVersionInfo(new SchemaVersionKey(schemaName, v1.getVersion())).getSchemaText(); Assert.assertEquals(schema1, receivedSchema); }
Assert.assertEquals(v1.getVersion() + 1, v2.getVersion().intValue()); SchemaVersionInfo schemaVersionInfo = schemaRegistryClient.getSchemaVersionInfo(new SchemaVersionKey(schemaName, v2 .getVersion())); SchemaVersionInfo latest = schemaRegistryClient.getLatestSchemaVersionInfo(schemaName);