@Override public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) { final RecordField field = new RecordField("root", RecordFieldType.RECORD.getRecordDataType(context.getRecord().getSchema())); final FieldValue fieldValue = new StandardFieldValue(context.getRecord(), field, null); return Stream.of(fieldValue); } }
/** * Converts an Avro Schema to a RecordSchema * * @param avroSchema the Avro Schema to convert * @param schemaText the textual representation of the schema * @param schemaId the identifier of the schema * @return the Corresponding Record Schema */ public static RecordSchema createSchema(final Schema avroSchema, final String schemaText, final SchemaIdentifier schemaId) { if (avroSchema == null) { throw new IllegalArgumentException("Avro Schema cannot be null"); } final String schemaFullName = avroSchema.getNamespace() + "." + avroSchema.getName(); final SimpleRecordSchema recordSchema = schemaText == null ? new SimpleRecordSchema(schemaId) : new SimpleRecordSchema(schemaText, AVRO_SCHEMA_FORMAT, schemaId); recordSchema.setSchemaName(avroSchema.getName()); recordSchema.setSchemaNamespace(avroSchema.getNamespace()); final DataType recordSchemaType = RecordFieldType.RECORD.getRecordDataType(recordSchema); final Map<String, DataType> knownRecords = new HashMap<>(); knownRecords.put(schemaFullName, recordSchemaType); final List<RecordField> recordFields = new ArrayList<>(avroSchema.getFields().size()); for (final Field field : avroSchema.getFields()) { final String fieldName = field.name(); final Schema fieldSchema = field.schema(); final DataType dataType = AvroTypeUtil.determineDataType(fieldSchema, knownRecords); final boolean nullable = isNullable(fieldSchema); addFieldToList(recordFields, field, fieldName, fieldSchema, dataType, nullable); } recordSchema.setFields(recordFields); return recordSchema; }
return RecordFieldType.RECORD.getRecordDataType(recordSchema);
private RecordSchema getAccountWithTransactionSchema() { final List<RecordField> accountFields = new ArrayList<>(); accountFields.add(new RecordField("id", RecordFieldType.INT.getDataType())); accountFields.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType())); final DataType transactionRecordType = RecordFieldType.RECORD.getRecordDataType(getTransactionSchema()); final DataType transactionsType = RecordFieldType.ARRAY.getArrayDataType(transactionRecordType); accountFields.add(new RecordField("transactions", transactionsType)); return new SimpleRecordSchema(accountFields); }
} else { SimpleRecordSchema recordSchema = new SimpleRecordSchema(SchemaIdentifier.EMPTY); DataType recordSchemaType = RecordFieldType.RECORD.getRecordDataType(recordSchema); knownRecordTypes.put(schemaFullName, recordSchemaType);
while ((record = reader.nextRecord()) != null) { Map<String, Object> contentMap = (Map<String, Object>) DataTypeUtils.convertRecordFieldtoObject(record, RecordFieldType.RECORD.getRecordDataType(record.getSchema())); Document document = new Document(); for (String name : schema.getFieldNames()) {
recordCount++; Map<String, Object> contentMap = (Map<String, Object>) DataTypeUtils.convertRecordFieldtoObject(r, RecordFieldType.RECORD.getRecordDataType(r.getSchema()));
.convertRecordFieldtoObject(record, RecordFieldType.RECORD.getRecordDataType(record.getSchema())); Insert insertQuery;
writer.beginRecordSet(); while ((record = reader.nextRecord()) != null) { Map<String, Object> recordMap = (Map<String, Object>) DataTypeUtils.convertRecordFieldtoObject(record, RecordFieldType.RECORD.getRecordDataType(record.getSchema()));
TestRunner runner = TestRunners.newTestRunner(new ForkRecord()); final DataType accountRecordType = RecordFieldType.RECORD.getRecordDataType(getAccountWithTransactionSchema()); final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountRecordType);
TestRunner runner = TestRunners.newTestRunner(new ForkRecord()); final DataType accountRecordType = RecordFieldType.RECORD.getRecordDataType(getAccountWithTransactionSchema()); final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountRecordType);
@Test public void testForkExtractNotAnArray() throws IOException, MalformedRecordException, InitializationException { TestRunner runner = TestRunners.newTestRunner(new ForkRecord()); final DataType accountRecordType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema()); final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountRecordType); final List<RecordField> fields = getDefaultFields(); fields.add(new RecordField("accounts", accountsType)); final RecordSchema schema = new SimpleRecordSchema(fields); final List<RecordField> fieldsWrite = getDefaultFields(); fieldsWrite.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType())); final RecordSchema schemaWrite = new SimpleRecordSchema(fieldsWrite); final JsonRecordReader readerService = new JsonRecordReader(schema); final MockRecordWriter writerService = new CustomRecordWriter("header", false, schemaWrite); runner.addControllerService("reader", readerService); runner.enableControllerService(readerService); runner.addControllerService("writer", writerService); runner.enableControllerService(writerService); runner.setProperty(ForkRecord.RECORD_READER, "reader"); runner.setProperty(ForkRecord.RECORD_WRITER, "writer"); runner.setProperty(ForkRecord.MODE, ForkRecord.MODE_EXTRACT); runner.setProperty(ForkRecord.INCLUDE_PARENT_FIELDS, "true"); runner.setProperty("my-path", "/country"); runner.enqueue(new File("src/test/resources/TestForkRecord/single-element-nested-array.json").toPath()); runner.run(1); runner.assertTransferCount(ForkRecord.REL_ORIGINAL, 1); runner.assertTransferCount(ForkRecord.REL_FORK, 1); final MockFlowFile mff = runner.getFlowFilesForRelationship(ForkRecord.REL_FORK).get(0); mff.assertAttributeEquals("record.count", "0"); }
@Test public void testForkExtractSimpleWithoutParentFields() throws IOException, MalformedRecordException, InitializationException { TestRunner runner = TestRunners.newTestRunner(new ForkRecord()); final DataType accountRecordType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema()); final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountRecordType); final List<RecordField> fields = getDefaultFields(); fields.add(new RecordField("accounts", accountsType)); final RecordSchema schema = new SimpleRecordSchema(fields); final JsonRecordReader readerService = new JsonRecordReader(schema); final MockRecordWriter writerService = new CustomRecordWriter("header", false, getAccountSchema()); runner.addControllerService("reader", readerService); runner.enableControllerService(readerService); runner.addControllerService("writer", writerService); runner.enableControllerService(writerService); runner.setProperty(ForkRecord.RECORD_READER, "reader"); runner.setProperty(ForkRecord.RECORD_WRITER, "writer"); runner.setProperty(ForkRecord.MODE, ForkRecord.MODE_EXTRACT); runner.setProperty("my-path", "/accounts"); runner.enqueue(new File("src/test/resources/TestForkRecord/single-element-nested-array.json").toPath()); runner.run(1); runner.assertTransferCount(ForkRecord.REL_ORIGINAL, 1); runner.assertTransferCount(ForkRecord.REL_FORK, 1); final MockFlowFile mff = runner.getFlowFilesForRelationship(ForkRecord.REL_FORK).get(0); mff.assertAttributeEquals("record.count", "2"); mff.assertContentEquals("header\n42,4750.89\n43,48212.38\n"); }
TestRunner runner = TestRunners.newTestRunner(new ForkRecord()); final DataType accountRecordType = RecordFieldType.RECORD.getRecordDataType(getAccountWithTransactionSchema()); final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountRecordType);
@Test public void testForkExtractSimpleWithParentFields() throws IOException, MalformedRecordException, InitializationException { TestRunner runner = TestRunners.newTestRunner(new ForkRecord()); final DataType accountRecordType = RecordFieldType.RECORD.getRecordDataType(getAccountSchema()); final DataType accountsType = RecordFieldType.ARRAY.getArrayDataType(accountRecordType); final List<RecordField> fields = getDefaultFields(); fields.add(new RecordField("accounts", accountsType)); final RecordSchema schema = new SimpleRecordSchema(fields); final List<RecordField> fieldsWrite = getDefaultFields(); fieldsWrite.add(new RecordField("balance", RecordFieldType.DOUBLE.getDataType())); final RecordSchema schemaWrite = new SimpleRecordSchema(fieldsWrite); final JsonRecordReader readerService = new JsonRecordReader(schema); final MockRecordWriter writerService = new CustomRecordWriter("header", false, schemaWrite); runner.addControllerService("reader", readerService); runner.enableControllerService(readerService); runner.addControllerService("writer", writerService); runner.enableControllerService(writerService); runner.setProperty(ForkRecord.RECORD_READER, "reader"); runner.setProperty(ForkRecord.RECORD_WRITER, "writer"); runner.setProperty(ForkRecord.MODE, ForkRecord.MODE_EXTRACT); runner.setProperty(ForkRecord.INCLUDE_PARENT_FIELDS, "true"); runner.setProperty("my-path", "/accounts"); runner.enqueue(new File("src/test/resources/TestForkRecord/single-element-nested-array.json").toPath()); runner.run(1); runner.assertTransferCount(ForkRecord.REL_ORIGINAL, 1); runner.assertTransferCount(ForkRecord.REL_FORK, 1); final MockFlowFile mff = runner.getFlowFilesForRelationship(ForkRecord.REL_FORK).get(0); mff.assertAttributeEquals("record.count", "2"); mff.assertContentEquals("header\n42,4750.89,John Doe,123 My Street,My City,MS,11111,USA\n43,48212.38,John Doe,123 My Street,My City,MS,11111,USA\n"); }
return RecordFieldType.RECORD.getRecordDataType(record.getSchema());
/** * Converts an Avro Schema to a RecordSchema * * @param avroSchema the Avro Schema to convert * @param schemaText the textual representation of the schema * @param schemaId the identifier of the schema * @return the Corresponding Record Schema */ public static RecordSchema createSchema(final Schema avroSchema, final String schemaText, final SchemaIdentifier schemaId) { if (avroSchema == null) { throw new IllegalArgumentException("Avro Schema cannot be null"); } final String schemaFullName = avroSchema.getNamespace() + "." + avroSchema.getName(); final SimpleRecordSchema recordSchema = schemaText == null ? new SimpleRecordSchema(schemaId) : new SimpleRecordSchema(schemaText, AVRO_SCHEMA_FORMAT, schemaId); final DataType recordSchemaType = RecordFieldType.RECORD.getRecordDataType(recordSchema); final Map<String, DataType> knownRecords = new HashMap<>(); knownRecords.put(schemaFullName, recordSchemaType); final List<RecordField> recordFields = new ArrayList<>(avroSchema.getFields().size()); for (final Field field : avroSchema.getFields()) { final String fieldName = field.name(); final Schema fieldSchema = field.schema(); final DataType dataType = AvroTypeUtil.determineDataType(fieldSchema, knownRecords); final boolean nullable = isNullable(fieldSchema); addFieldToList(recordFields, field, fieldName, fieldSchema, dataType, nullable); } recordSchema.setFields(recordFields); return recordSchema; }
@Override public Stream<FieldValue> evaluate(final RecordPathEvaluationContext context) { final RecordField field = new RecordField("root", RecordFieldType.RECORD.getRecordDataType(context.getRecord().getSchema())); final FieldValue fieldValue = new StandardFieldValue(context.getRecord(), field, null); return Stream.of(fieldValue); } }
return RecordFieldType.RECORD.getRecordDataType(recordSchema);
.convertRecordFieldtoObject(record, RecordFieldType.RECORD.getRecordDataType(record.getSchema())); Insert insertQuery;