@Override public Map<String, String> getAttributes(final RecordSchema schema) { // First, check if schema has the Avro Text available already. final Optional<String> schemaFormat = schema.getSchemaFormat(); if (schemaFormat.isPresent() && AvroTypeUtil.AVRO_SCHEMA_FORMAT.equals(schemaFormat.get())) { final Optional<String> schemaText = schema.getSchemaText(); if (schemaText.isPresent()) { return Collections.singletonMap("avro.schema", schemaText.get()); } } return Collections.singletonMap("avro.schema", avroSchemaTextCache.get(schema)); }
public static Schema extractAvroSchema(final RecordSchema recordSchema) { if (recordSchema == null) { throw new IllegalArgumentException("RecordSchema cannot be null"); } final Optional<String> schemaFormatOption = recordSchema.getSchemaFormat(); if (!schemaFormatOption.isPresent()) { return buildAvroSchema(recordSchema); } final String schemaFormat = schemaFormatOption.get(); if (!schemaFormat.equals(AVRO_SCHEMA_FORMAT)) { return buildAvroSchema(recordSchema); } final Optional<String> textOption = recordSchema.getSchemaText(); if (!textOption.isPresent()) { return buildAvroSchema(recordSchema); } final String text = textOption.get(); return new Schema.Parser().parse(text); }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves and returns the textual representation of the schema based on * the provided name of the schema available in Schema Registry. * * @return the text that corresponds to the latest version of the schema with the given name * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given name */ default String retrieveSchemaText(String schemaName) throws IOException, SchemaNotFoundException { final RecordSchema recordSchema = retrieveSchema(SchemaIdentifier.builder().name(schemaName).build()); if (recordSchema == null) { throw new SchemaNotFoundException("Could not find schema with name '" + schemaName + "'"); } return recordSchema.getSchemaText().get(); }
protected String getGroupId(final ProcessContext context, final FlowFile flowFile, final RecordSchema schema, final ProcessSession session) { final String mergeStrategy = context.getProperty(MERGE_STRATEGY).getValue(); if (MERGE_STRATEGY_DEFRAGMENT.equals(mergeStrategy)) { return flowFile.getAttribute(FRAGMENT_ID_ATTRIBUTE); } final Optional<String> optionalText = schema.getSchemaText(); final String schemaText = optionalText.isPresent() ? optionalText.get() : AvroTypeUtil.extractAvroSchema(schema).toString(); final String groupId; final String correlationshipAttributeName = context.getProperty(CORRELATION_ATTRIBUTE_NAME).getValue(); if (correlationshipAttributeName != null) { final String correlationAttr = flowFile.getAttribute(correlationshipAttributeName); groupId = correlationAttr == null ? schemaText : schemaText + correlationAttr; } else { groupId = schemaText; } return groupId; }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves the textual representation of the schema with the given ID and version * * @param schemaId the unique identifier for the desired schema * @param version the version of the desired schema * @return the textual representation of the schema with the given ID and version * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given id and version */ default String retrieveSchemaText(long schemaId, int version) throws IOException, SchemaNotFoundException { final RecordSchema recordSchema = retrieveSchema(SchemaIdentifier.builder().id(schemaId).version(version).build()); if (recordSchema == null) { throw new SchemaNotFoundException("Could not find schema with ID '" + schemaId + "' and version '" + version + "'"); } return recordSchema.getSchemaText().get(); }
@Override public RecordReader createRecordReader(final Map<String, String> variables, final InputStream in, final ComponentLog logger) throws IOException, SchemaNotFoundException { final String schemaAccessStrategy = getConfigurationContext().getProperty(getSchemaAcessStrategyDescriptor()).getValue(); if (EMBEDDED_AVRO_SCHEMA.getValue().equals(schemaAccessStrategy)) { return new AvroReaderWithEmbeddedSchema(in); } else { final RecordSchema recordSchema = getSchema(variables, in, null); final Schema avroSchema; try { if (recordSchema.getSchemaFormat().isPresent() & recordSchema.getSchemaFormat().get().equals(AvroTypeUtil.AVRO_SCHEMA_FORMAT)) { final Optional<String> textOption = recordSchema.getSchemaText(); if (textOption.isPresent()) { avroSchema = compiledAvroSchemaCache.get(textOption.get()); } else { avroSchema = AvroTypeUtil.extractAvroSchema(recordSchema); } } else { avroSchema = AvroTypeUtil.extractAvroSchema(recordSchema); } } catch (final Exception e) { throw new SchemaNotFoundException("Failed to compile Avro Schema", e); } return new AvroReaderWithExplicitSchema(in, recordSchema, avroSchema); } }
@Override public RecordSetWriter createWriter(final ComponentLog logger, final RecordSchema recordSchema, final OutputStream out) throws IOException { final String strategyValue = getConfigurationContext().getProperty(getSchemaWriteStrategyDescriptor()).getValue(); final String compressionFormat = getConfigurationContext().getProperty(COMPRESSION_FORMAT).getValue(); try { final Schema avroSchema; try { if (recordSchema.getSchemaFormat().isPresent() && recordSchema.getSchemaFormat().get().equals(AvroTypeUtil.AVRO_SCHEMA_FORMAT)) { final Optional<String> textOption = recordSchema.getSchemaText(); if (textOption.isPresent()) { avroSchema = compiledAvroSchemaCache.get(textOption.get()); } else { avroSchema = AvroTypeUtil.extractAvroSchema(recordSchema); } } else { avroSchema = AvroTypeUtil.extractAvroSchema(recordSchema); } } catch (final Exception e) { throw new SchemaNotFoundException("Failed to compile Avro Schema", e); } if (AVRO_EMBEDDED.getValue().equals(strategyValue)) { return new WriteAvroResultWithSchema(avroSchema, out, getCodecFactory(compressionFormat)); } else { return new WriteAvroResultWithExternalSchema(avroSchema, recordSchema, getSchemaAccessWriter(recordSchema), out, encoderPool, getLogger()); } } catch (final SchemaNotFoundException e) { throw new ProcessException("Could not determine the Avro Schema to use for writing the content", e); } }
@Override public Map<String, String> getAttributes(final RecordSchema schema) { // First, check if schema has the Avro Text available already. final Optional<String> schemaFormat = schema.getSchemaFormat(); if (schemaFormat.isPresent() && AvroTypeUtil.AVRO_SCHEMA_FORMAT.equals(schemaFormat.get())) { final Optional<String> schemaText = schema.getSchemaText(); if (schemaText.isPresent()) { return Collections.singletonMap("avro.schema", schemaText.get()); } } String schemaText; synchronized (avroSchemaTextCache) { schemaText = avroSchemaTextCache.get(schema); } if (schemaText == null) { final Schema avroSchema = AvroTypeUtil.extractAvroSchema(schema); schemaText = avroSchema.toString(); synchronized (avroSchemaTextCache) { final String existing = avroSchemaTextCache.putIfAbsent(schema, schemaText); if (existing != null) { schemaText = existing; } } } return Collections.singletonMap("avro.schema", schemaText); }
public static Schema extractAvroSchema(final RecordSchema recordSchema) { if (recordSchema == null) { throw new IllegalArgumentException("RecordSchema cannot be null"); } final Optional<String> schemaFormatOption = recordSchema.getSchemaFormat(); if (!schemaFormatOption.isPresent()) { return buildAvroSchema(recordSchema); } final String schemaFormat = schemaFormatOption.get(); if (!schemaFormat.equals(AVRO_SCHEMA_FORMAT)) { return buildAvroSchema(recordSchema); } final Optional<String> textOption = recordSchema.getSchemaText(); if (!textOption.isPresent()) { return buildAvroSchema(recordSchema); } final String text = textOption.get(); return new Schema.Parser().parse(text); }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves and returns the textual representation of the schema based on * the provided name of the schema available in Schema Registry. * * @return the text that corresponds to the latest version of the schema with the given name * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given name */ default String retrieveSchemaText(String schemaName) throws IOException, SchemaNotFoundException { final RecordSchema recordSchema = retrieveSchema(SchemaIdentifier.builder().name(schemaName).build()); if (recordSchema == null) { throw new SchemaNotFoundException("Could not find schema with name '" + schemaName + "'"); } return recordSchema.getSchemaText().get(); }
protected String getGroupId(final ProcessContext context, final FlowFile flowFile, final RecordSchema schema, final ProcessSession session) { final String mergeStrategy = context.getProperty(MERGE_STRATEGY).getValue(); if (MERGE_STRATEGY_DEFRAGMENT.equals(mergeStrategy)) { return flowFile.getAttribute(FRAGMENT_ID_ATTRIBUTE); } final Optional<String> optionalText = schema.getSchemaText(); final String schemaText = optionalText.isPresent() ? optionalText.get() : AvroTypeUtil.extractAvroSchema(schema).toString(); final String groupId; final String correlationshipAttributeName = context.getProperty(CORRELATION_ATTRIBUTE_NAME).getValue(); if (correlationshipAttributeName != null) { final String correlationAttr = flowFile.getAttribute(correlationshipAttributeName); groupId = correlationAttr == null ? schemaText : schemaText + correlationAttr; } else { groupId = schemaText; } return groupId; }
/** * @deprecated Use {@link #retrieveSchema(SchemaIdentifier)} instead * * Retrieves the textual representation of the schema with the given ID and version * * @param schemaId the unique identifier for the desired schema * @param version the version of the desired schema * @return the textual representation of the schema with the given ID and version * * @throws IOException if unable to communicate with the backing store * @throws SchemaNotFoundException if unable to find the schema with the given id and version */ default String retrieveSchemaText(long schemaId, int version) throws IOException, SchemaNotFoundException { final RecordSchema recordSchema = retrieveSchema(SchemaIdentifier.builder().id(schemaId).version(version).build()); if (recordSchema == null) { throw new SchemaNotFoundException("Could not find schema with ID '" + schemaId + "' and version '" + version + "'"); } return recordSchema.getSchemaText().get(); }