protected static SchemaBuilder schemaBuilder() { return SchemaBuilder.struct() .field(DEBEZIUM_VERSION_KEY, Schema.OPTIONAL_STRING_SCHEMA) .field(DEBEZIUM_CONNECTOR_KEY, Schema.OPTIONAL_STRING_SCHEMA); }
public static Schema removeImplicitRowTimeRowKeyFromSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { String fieldName = field.name(); fieldName = fieldName.substring(fieldName.indexOf('.') + 1); if (!fieldName.equalsIgnoreCase(SchemaUtil.ROWTIME_NAME) && !fieldName.equalsIgnoreCase(SchemaUtil.ROWKEY_NAME)) { schemaBuilder.field(fieldName, field.schema()); } } return schemaBuilder.build(); }
/** * Rename field names to be consistent with the internal column names. */ public static Schema getAvroSerdeKsqlSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { schemaBuilder.field(field.name().replace(".", "_"), field.schema()); } return schemaBuilder.build(); }
/** * Remove the alias when reading/writing from outside */ public static Schema getSchemaWithNoAlias(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { final String name = getFieldNameWithNoAlias(field); schemaBuilder.field(name, field.schema()); } return schemaBuilder.build(); }
public static Schema addImplicitRowTimeRowKeyToSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); schemaBuilder.field(SchemaUtil.ROWTIME_NAME, Schema.OPTIONAL_INT64_SCHEMA); schemaBuilder.field(SchemaUtil.ROWKEY_NAME, Schema.OPTIONAL_STRING_SCHEMA); for (final Field field : schema.fields()) { if (!field.name().equals(SchemaUtil.ROWKEY_NAME) && !field.name().equals(SchemaUtil.ROWTIME_NAME)) { schemaBuilder.field(field.name(), field.schema()); } } return schemaBuilder.build(); }
private Schema toKsqlStructSchema(final Schema schema) { final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { try { final Schema fieldSchema = toKsqlFieldSchema(field.schema()); schemaBuilder.field(field.name().toUpperCase(), fieldSchema); } catch (final UnsupportedTypeException e) { log.error("Error inferring schema at field {}: {}", field.name(), e.getMessage()); } } return schemaBuilder.optional().build(); } }
private void addTestTopicAndSources() { final Schema schema1 = SchemaBuilder.struct().field("S1_F1", Schema.OPTIONAL_BOOLEAN_SCHEMA); ensureSource( DataSource.DataSourceType.KTABLE, "TEST_TABLE", "KAFKA_TOPIC_1", "KSQL_TOPIC_1", schema1); final Schema schema2 = SchemaBuilder.struct().field("S2_F1", Schema.OPTIONAL_STRING_SCHEMA); ensureSource( DataSource.DataSourceType.KSTREAM, "TEST_STREAM", "KAFKA_TOPIC_2", "KSQL_TOPIC_2", schema2); givenTopicExists("orders-topic"); }
public static Schema buildSchemaWithAlias(final Schema schema, final String alias) { final SchemaBuilder newSchema = SchemaBuilder.struct().name(schema.name()); for (final Field field : schema.fields()) { newSchema.field((alias + "." + field.name()), field.schema()); } return newSchema; }
@Before public void before() { orderSchema = SchemaBuilder.struct() .field("ordertime".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("orderid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("itemid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_STRING_SCHEMA) .field("orderunits".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA) .build(); }
private static Schema buildStructSchema(final Struct struct) { final SchemaBuilder structSchemaBuilder = SchemaBuilder.struct(); for (final Pair<String, Type> field: struct.getItems()) { structSchemaBuilder.field(field.getLeft(), getTypeSchema(field.getRight())); } return structSchemaBuilder.optional().build(); } }
@Test public void shouldTranslateNullValueCorrectly() { final Schema rowSchema = SchemaBuilder.struct() .field("INT", SchemaBuilder.OPTIONAL_INT32_SCHEMA) .optional() .build(); final Struct connectStruct = new Struct(rowSchema); final ConnectDataTranslator connectToKsqlTranslator = new ConnectDataTranslator(rowSchema); final GenericRow row = connectToKsqlTranslator.toKsqlRow(rowSchema, connectStruct); assertThat(row.getColumns().size(), equalTo(1)); assertThat(row.getColumnValue(0), is(nullValue())); }
@Before public void before() { orderSchema = SchemaBuilder.struct() .field("ordertime".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("orderid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA) .field("itemid".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_STRING_SCHEMA) .field("orderunits".toUpperCase(), org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA) .build(); delimitedDeserializer = new KsqlDelimitedDeserializer(orderSchema, recordLogger); }
@Test public void shouldIgnoreUnsupportedType() { final Schema connectSchema = SchemaBuilder .struct() .field("bytesField", Schema.BYTES_SCHEMA) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.fields().size(), equalTo(0)); } }
@Test public void shouldBuildCorrectMapField() { final Schema schema = SchemaBuilder .struct() .field("field", SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.INT32_SCHEMA)) .build(); final List<FieldInfo> entity = EntityUtil.buildSourceSchemaEntity(schema); assertThat(entity.size(), equalTo(1)); assertThat(entity.get(0).getName(), equalTo("field")); assertThat(entity.get(0).getSchema().getTypeName(), equalTo("MAP")); assertThat(entity.get(0).getSchema().getFields(), equalTo(Optional.empty())); assertThat(entity.get(0).getSchema().getMemberSchema().get().getTypeName(), equalTo("INTEGER")); }
@Test public void shouldBuildCorrectArrayField() { final Schema schema = SchemaBuilder .struct() .field("field", SchemaBuilder.array(SchemaBuilder.INT64_SCHEMA)) .build(); final List<FieldInfo> entity = EntityUtil.buildSourceSchemaEntity(schema); assertThat(entity.size(), equalTo(1)); assertThat(entity.get(0).getName(), equalTo("field")); assertThat(entity.get(0).getSchema().getTypeName(), equalTo("ARRAY")); assertThat(entity.get(0).getSchema().getFields(), equalTo(Optional.empty())); assertThat(entity.get(0).getSchema().getMemberSchema().get().getTypeName(), equalTo("BIGINT")); }
private static List<FieldInfo> buildTestSchema(final int size) { final SchemaBuilder dataSourceBuilder = SchemaBuilder.struct().name("TestSchema"); for (int i = 0; i < size; i++) { dataSourceBuilder.field("f_" + i, SchemaUtil.getTypeSchema("STRING")); } return EntityUtil.buildSourceSchemaEntity(dataSourceBuilder.build()); } }
@Test public void shoudlReplacePrimitivesCorrectly() { final Schema schema = SchemaBuilder.struct() .field("COLUMN_NAME", Schema.OPTIONAL_INT64_SCHEMA) .optional() .build(); final AvroDataTranslator dataTranslator = new AvroDataTranslator(schema, KsqlConstants.DEFAULT_AVRO_SCHEMA_FULL_NAME); final GenericRow ksqlRow = new GenericRow(Collections.singletonList(123L)); final Struct struct = dataTranslator.toConnectRow(ksqlRow); assertThat(struct.get("COLUMN_NAME"), equalTo(123L)); final GenericRow translatedRow = dataTranslator.toKsqlRow(struct.schema(), struct); assertThat(translatedRow, equalTo(ksqlRow)); }
@Test public void shouldTranslateMaps() { final Schema connectSchema = SchemaBuilder .struct() .field("mapField", SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
private StructuredDataSource buildDataSource(final String kafkaTopicName) { final Schema schema = SchemaBuilder.struct() .field("field0", Schema.OPTIONAL_INT32_SCHEMA) .build(); final KsqlTopic topic = new KsqlTopic("internal", kafkaTopicName, new KsqlJsonTopicSerDe(), true); return new KsqlStream<>( "query", "stream", schema, schema.fields().get(0), new MetadataTimestampExtractionPolicy(), topic, Serdes.String()); }
@Test public void shouldGetCorrectSqlTypeNameForStruct() { final Schema structSchema = SchemaBuilder.struct() .field("COL1", Schema.OPTIONAL_STRING_SCHEMA) .field("COL2", Schema.OPTIONAL_INT32_SCHEMA) .field("COL3", Schema.OPTIONAL_FLOAT64_SCHEMA) .field("COL4", SchemaBuilder.array(Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()) .field("COL5", SchemaBuilder.map(Schema.OPTIONAL_STRING_SCHEMA, Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build()) .build(); assertThat(SchemaUtil.getSqlTypeName(structSchema), equalTo( "STRUCT<COL1 VARCHAR, COL2 INT, COL3 DOUBLE, COL4 ARRAY<DOUBLE>, COL5 MAP<VARCHAR,DOUBLE>>")); }