private SchemaVersionInfo getSchemaVersionInfo(final SchemaRegistryClient client, final SchemaVersionKey key) throws org.apache.nifi.schema.access.SchemaNotFoundException { try { // Try to fetch the SchemaVersionInfo from the cache. final Tuple<SchemaVersionInfo, Long> timestampedVersionInfo = schemaVersionByKeyCache.get(key); // Determine if the timestampedVersionInfo is expired boolean fetch = false; if (timestampedVersionInfo == null) { fetch = true; } else { final long minTimestamp = System.nanoTime() - versionInfoCacheNanos; fetch = timestampedVersionInfo.getValue() < minTimestamp; } // If not expired, use what we got from the cache if (!fetch) { return timestampedVersionInfo.getKey(); } // schema version info was expired or not found in cache. Fetch from schema registry final SchemaVersionInfo versionInfo = client.getSchemaVersionInfo(key); if (versionInfo == null) { throw new org.apache.nifi.schema.access.SchemaNotFoundException("Could not find schema with name '" + key.getSchemaName() + "' and version " + key.getVersion()); } // Store new version in cache. final Tuple<SchemaVersionInfo, Long> tuple = new Tuple<>(versionInfo, System.nanoTime()); schemaVersionByKeyCache.put(key, tuple); return versionInfo; } catch (final SchemaNotFoundException e) { throw new org.apache.nifi.schema.access.SchemaNotFoundException(e); } }
final SchemaMetadataInfo info = client.getSchemaMetadataInfo(schemaId.getAsLong()); if (info == null) { throw new org.apache.nifi.schema.access.SchemaNotFoundException("Could not find schema with ID '" + schemaId + "' and version '" + version + "'"); final SchemaMetadata metadata = info.getSchemaMetadata(); schemaName = metadata.getName(); final SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaName, version.getAsInt()); versionInfo = getSchemaVersionInfo(client, schemaVersionKey); if (versionInfo == null) { final String schemaText = versionInfo.getSchemaText();
@OnEnabled public void enable(final ConfigurationContext context) throws InitializationException { schemaRegistryConfig = new HashMap<>(); versionInfoCacheNanos = context.getProperty(CACHE_EXPIRATION).asTimePeriod(TimeUnit.NANOSECONDS); // The below properties may or may not need to be exposed to the end // user. We just need to watch usage patterns to see if sensible default // can satisfy NiFi requirements String urlValue = context.getProperty(URL).evaluateAttributeExpressions().getValue(); if (urlValue == null || urlValue.trim().isEmpty()) { throw new IllegalArgumentException("'Schema Registry URL' must not be null or empty."); } schemaRegistryConfig.put(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), urlValue); schemaRegistryConfig.put(SchemaRegistryClient.Configuration.CLASSLOADER_CACHE_SIZE.name(), 10L); schemaRegistryConfig.put(SchemaRegistryClient.Configuration.CLASSLOADER_CACHE_EXPIRY_INTERVAL_SECS.name(), context.getProperty(CACHE_EXPIRATION).asTimePeriod(TimeUnit.SECONDS)); schemaRegistryConfig.put(SchemaRegistryClient.Configuration.SCHEMA_VERSION_CACHE_SIZE.name(), context.getProperty(CACHE_SIZE).asInteger()); schemaRegistryConfig.put(SchemaRegistryClient.Configuration.SCHEMA_VERSION_CACHE_EXPIRY_INTERVAL_SECS.name(), context.getProperty(CACHE_EXPIRATION).asTimePeriod(TimeUnit.SECONDS)); }
public SchemaMetadata toSchemaMetadata() { return new SchemaMetadata.Builder(getName()) .type(getType()) .schemaGroup(getSchemaGroup()) .compatibility(getCompatibility()) .validationLevel(getValidationLevel()) .description(getDescription()) .evolve(getEvolve()) .build(); }
final SchemaMetadataInfo metadataInfo = client.getSchemaMetadataInfo(schemaName.get()); if (metadataInfo == null) { throw new org.apache.nifi.schema.access.SchemaNotFoundException("Could not find schema with name '" + schemaName + "'"); schemaId = metadataInfo.getId(); if (schemaId == null) { throw new org.apache.nifi.schema.access.SchemaNotFoundException("Could not find schema with name '" + schemaName + "'"); final SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaName.get(), schemaVersion.getAsInt()); versionInfo = getSchemaVersionInfo(client, schemaVersionKey); } else { if (versionInfo == null || versionInfo.getVersion() == null) { final String message = createErrorMessage("Could not find schema", schemaName, schemaBranchName, schemaVersion); throw new org.apache.nifi.schema.access.SchemaNotFoundException(message); final String schemaText = versionInfo.getSchemaText(); .name(schemaName.get()) .branch(schemaBranchName.orElse(null)) .version(versionInfo.getVersion()) .build();
private SchemaVersionInfo retrieveSchemaVersionInfo(SchemaIdVersion key) throws SchemaNotFoundException { SchemaVersionInfo schemaVersionInfo = null; if (key.getSchemaVersionId() != null) { schemaVersionInfo = fetchSchemaVersionInfo(key.getSchemaVersionId()); } else if (key.getSchemaMetadataId() != null) { SchemaMetadataInfo schemaMetadataInfo = schemaMetadataFetcher.getSchemaMetadataInfo(key.getSchemaMetadataId()); Integer version = key.getVersion(); schemaVersionInfo = fetchSchemaVersionInfo(schemaMetadataInfo.getSchemaMetadata().getName(), version); } else { throw new IllegalArgumentException("Invalid SchemaIdVersion: " + key); } return schemaVersionInfo; }
public static SchemaMetadataStorable updateSchemaMetadata(SchemaMetadataStorable schemaMetadataStorable, SchemaMetadata schemaMetadata) { schemaMetadataStorable.setType(schemaMetadata.getType()); schemaMetadataStorable.setSchemaGroup(schemaMetadata.getSchemaGroup()); schemaMetadataStorable.setName(schemaMetadata.getName()); schemaMetadataStorable.setDescription(schemaMetadata.getDescription()); schemaMetadataStorable.setCompatibility(schemaMetadata.getCompatibility()); schemaMetadataStorable.setValidationLevel(schemaMetadata.getValidationLevel()); schemaMetadataStorable.setEvolve(schemaMetadata.isEvolve()); return schemaMetadataStorable; }
private AggregatedSchemaMetadataInfo buildAggregatedSchemaMetadataInfo(SchemaMetadataInfo schemaMetadataInfo) throws SchemaNotFoundException, SchemaBranchNotFoundException { if (schemaMetadataInfo == null) { return null; } List<SerDesInfo> serDesInfos = getSerDesInfos(schemaMetadataInfo.getSchemaMetadata().getName()); return new AggregatedSchemaMetadataInfo(schemaMetadataInfo.getSchemaMetadata(), schemaMetadataInfo.getId(), schemaMetadataInfo.getTimestamp(), getAggregatedSchemaBranch(schemaMetadataInfo.getSchemaMetadata().getName()), serDesInfos); }
public SchemaIdVersion addSchemaVersion(SchemaMetadata schemaMetadata, SchemaVersion schemaVersion) throws IncompatibleSchemaException, InvalidSchemaException, SchemaNotFoundException, SchemaBranchNotFoundException { lockSchemaMetadata(schemaMetadata.getName()); return schemaVersionLifecycleManager.addSchemaVersion(SchemaBranch.MASTER_BRANCH, schemaMetadata, schemaVersion, x -> registerSchemaMetadata(x)); }
public static SchemaMetadataStorable fromSchemaMetadataInfo(SchemaMetadataInfo schemaMetadataInfo) { SchemaMetadata schemaMetadata = schemaMetadataInfo.getSchemaMetadata(); SchemaMetadataStorable schemaMetadataStorable = updateSchemaMetadata(new SchemaMetadataStorable(), schemaMetadata); schemaMetadataStorable.setId(schemaMetadataInfo.getId()); schemaMetadataStorable.setTimestamp(schemaMetadataInfo.getTimestamp()); return schemaMetadataStorable; }
public static SchemaFieldInfoStorable fromSchemaFieldInfo(SchemaFieldInfo schemaFieldInfo, Long id) { SchemaFieldInfoStorable schemaFieldInfoStorable = new SchemaFieldInfoStorable(id); schemaFieldInfoStorable.setFieldNamespace(schemaFieldInfo.getNamespace()); schemaFieldInfoStorable.setName(schemaFieldInfo.getName()); schemaFieldInfoStorable.setType(schemaFieldInfo.getType()); return schemaFieldInfoStorable; }
@Override public CompatibilityResult validate(T toSchema, T existingSchema) { CompatibilityResult compatibilityResult = schemaValidator.validate(toSchema, existingSchema); if (compatibilityResult.isCompatible()) { compatibilityResult = schemaValidator.validate(existingSchema, toSchema); } return compatibilityResult; } }
@Override public SchemaMetadataInfo getSchemaMetadata(long schemaVersionId) throws SchemaNotFoundException { SchemaVersionInfo schemaVersionInfo = getSchemaVersionInfo(schemaVersionId); return getSchemaMetadataInfo(schemaVersionInfo.getName()); }
public SerDesInfo toSerDesInfo() { return new SerDesInfo(id, timestamp, new SerDesPair(name, description, fileId, serializerClassName, deserializerClassName)); }
public SchemaIdVersion addSchemaVersion(String schemaBranchName, String schemaName, SchemaVersion schemaVersion) throws SchemaNotFoundException, IncompatibleSchemaException, InvalidSchemaException, SchemaBranchNotFoundException { lockSchemaMetadata(schemaName); return schemaVersionLifecycleManager.addSchemaVersion(schemaBranchName, schemaName, schemaVersion); }
public SchemaIdVersion addSchemaVersion(String schemaBranchName, SchemaMetadata schemaMetadata, SchemaVersion schemaVersion) throws IncompatibleSchemaException, InvalidSchemaException, SchemaNotFoundException, SchemaBranchNotFoundException { lockSchemaMetadata(schemaMetadata.getName()); return schemaVersionLifecycleManager.addSchemaVersion(schemaBranchName, schemaMetadata, schemaVersion, x -> registerSchemaMetadata(x)); }
@Override public CompatibilityResult validate(T toSchema, T existingSchema) { CompatibilityResult compatibilityResult = schemaValidator.validate(toSchema, existingSchema); if (compatibilityResult.isCompatible()) { compatibilityResult = schemaValidator.validate(existingSchema, toSchema); } return compatibilityResult; } }
public SchemaIdVersion addSchemaVersion(String schemaName, SchemaVersion schemaVersion) throws SchemaNotFoundException, IncompatibleSchemaException, InvalidSchemaException, SchemaBranchNotFoundException { lockSchemaMetadata(schemaName); return schemaVersionLifecycleManager.addSchemaVersion(SchemaBranch.MASTER_BRANCH, schemaName, schemaVersion); }