private List<FunctionParameter> mapToFunctionParameter(final List<Schema> params) { return params .stream() .map(schema -> schema == null ? new FunctionParameter(null, false) : new FunctionParameter(schema.type(), schema.isOptional())) .collect(Collectors.toList()); }
protected Object convertTimestampToLocalDateTime(Column column, Field fieldDefn, Object data) { if (data == null && !fieldDefn.schema().isOptional()) { return null; } if (!(data instanceof Timestamp)) { return data; } return ((Timestamp)data).toLocalDateTime(); } }
/** * Convert an unknown data value. * * @param column the column definition describing the {@code data} value; never null * @param fieldDefn the field definition; never null * @param data the data object to be converted into a {@link Date Kafka Connect date} type; never null * @return the converted value, or null if the conversion could not be made and the column allows nulls * @throws IllegalArgumentException if the value could not be converted but the column does not allow nulls */ protected Object handleUnknownData(Column column, Field fieldDefn, Object data) { if (column.isOptional() || fieldDefn.schema().isOptional()) { Class<?> dataClass = data.getClass(); logger.warn("Unexpected value for JDBC type {} and column {}: class={}", column.jdbcType(), column, dataClass.isArray() ? dataClass.getSimpleName() : dataClass.getName()); // don't include value in case its // sensitive return null; } throw new IllegalArgumentException("Unexpected value for JDBC type " + column.jdbcType() + " and column " + column + ": class=" + data.getClass()); // don't include value in case its sensitive }
appendFirst("type", schema.type()); appendAdditional("optional", schema.isOptional()); if (schema.doc() != null) appendAdditional("doc", schema.doc()); if (schema.version() != null) appendAdditional("version", schema.version());
@Test public void shouldTranslateArray() { final Schema connectSchema = SchemaBuilder .struct() .field("arrayField", SchemaBuilder.array(Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("ARRAYFIELD"), notNullValue()); final Schema arraySchema = ksqlSchema.field("ARRAYFIELD").schema(); assertThat(arraySchema.type(), equalTo(Schema.Type.ARRAY)); assertThat(arraySchema.isOptional(), is(true)); assertThat(arraySchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void shouldTranslateNested() { final Schema connectInnerSchema = SchemaBuilder .struct() .field("intField", Schema.INT32_SCHEMA) .build(); final Schema connectSchema = SchemaBuilder .struct() .field("structField", connectInnerSchema) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("STRUCTFIELD"), notNullValue()); final Schema innerSchema = ksqlSchema.field("STRUCTFIELD").schema(); assertThat(innerSchema.fields().size(), equalTo(connectInnerSchema.fields().size())); for (int i = 0; i < connectInnerSchema.fields().size(); i++) { assertThat( innerSchema.fields().get(i).name().toUpperCase(), equalTo(connectInnerSchema.fields().get(i).name().toUpperCase())); assertThat( innerSchema.fields().get(i).schema().type(), equalTo(connectInnerSchema.fields().get(i).schema().type())); assertThat(innerSchema.fields().get(i).schema().isOptional(), is(true)); } }
@Test public void shouldTranslatePrimitives() { final Schema connectSchema = SchemaBuilder .struct() .field("intField", Schema.INT32_SCHEMA) .field("longField", Schema.INT64_SCHEMA) .field("doubleField", Schema.FLOAT64_SCHEMA) .field("stringField", Schema.STRING_SCHEMA) .field("booleanField", Schema.BOOLEAN_SCHEMA) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.schema().type(), equalTo(Schema.Type.STRUCT)); assertThat(ksqlSchema.fields().size(), equalTo(connectSchema.fields().size())); for (int i = 0; i < ksqlSchema.fields().size(); i++) { assertThat( ksqlSchema.fields().get(i).name(), equalTo(connectSchema.fields().get(i).name().toUpperCase())); assertThat( ksqlSchema.fields().get(i).schema().type(), equalTo(connectSchema.fields().get(i).schema().type())); assertThat(ksqlSchema.fields().get(i).schema().isOptional(), is(true)); } }
boolean equal = Objects.equals(schema1.isOptional(), schema2.isOptional()) && Objects.equals(schema1.version(), schema2.version()) && Objects.equals(schema1.name(), schema2.name()) &&
private SchemaBuilder copySchemaExcludingName(Schema source, SchemaBuilder builder, boolean copyFields) { builder.version(source.version()); builder.doc(source.doc()); Map<String, String> params = source.parameters(); if (params != null) { builder.parameters(params); } if (source.isOptional()) { builder.optional(); } else { builder.required(); } if (copyFields) { for (org.apache.kafka.connect.data.Field field : source.fields()) { builder.field(field.name(), field.schema()); } } return builder; } }
@Test public void shouldTranslateStructInsideMap() { final Schema connectSchema = SchemaBuilder .struct() .field( "mapField", SchemaBuilder.map( Schema.STRING_SCHEMA, SchemaBuilder.struct() .field("innerIntField", Schema.INT32_SCHEMA) .build())) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.isOptional(), is(true)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema().type(), equalTo(Schema.Type.STRUCT)); assertThat(mapSchema.valueSchema().fields().size(), equalTo(1)); assertThat(mapSchema.valueSchema().fields().get(0).name(), equalTo("INNERINTFIELD")); assertThat(mapSchema.valueSchema().fields().get(0).schema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void unsignedSmallIntTest() throws InterruptedException { config = DATABASE.defaultConfig() .with(MySqlConnectorConfig.SNAPSHOT_MODE, MySqlConnectorConfig.SnapshotMode.INITIAL) .build(); start(MySqlConnector.class, config); // Testing.Print.enable(); SourceRecords records = consumeRecordsByTopic(EVENT_COUNT); SourceRecord record = records.recordsForTopic(DATABASE.topicForTable("UNSIGNED_SMALLINT_TABLE")).get(0); validate(record); Schema schemaA = record.valueSchema().fields().get(1).schema().fields().get(0).schema(); Schema schemaB = record.valueSchema().fields().get(1).schema().fields().get(1).schema(); Schema schemaC = record.valueSchema().fields().get(1).schema().fields().get(2).schema(); Schema schemaD = record.valueSchema().fields().get(1).schema().fields().get(3).schema(); Schema schemaE = record.valueSchema().fields().get(1).schema().fields().get(4).schema(); Schema schemaF = record.valueSchema().fields().get(1).schema().fields().get(5).schema(); assertThat(schemaA.isOptional()).isEqualTo(true); assertThat(schemaA.defaultValue()).isEqualTo(0); assertThat(schemaB.isOptional()).isEqualTo(true); assertThat(schemaB.defaultValue()).isEqualTo(10); assertThat(schemaC.isOptional()).isEqualTo(true); assertThat(schemaC.defaultValue()).isEqualTo(null); assertThat(schemaD.isOptional()).isEqualTo(false); assertThat(schemaE.isOptional()).isEqualTo(false); assertThat(schemaE.defaultValue()).isEqualTo(0); assertThat(schemaF.isOptional()).isEqualTo(false); assertThat(schemaF.defaultValue()).isEqualTo(0); assertEmptyFieldValue(record, "G"); }
Schema schemaE = record.valueSchema().fields().get(1).schema().fields().get(4).schema(); Schema schemaF = record.valueSchema().fields().get(1).schema().fields().get(5).schema(); assertThat(schemaA.isOptional()).isEqualTo(true); assertThat(schemaA.defaultValue()).isEqualTo(BigDecimal.ZERO); assertThat(schemaB.isOptional()).isEqualTo(true); assertThat(schemaB.defaultValue()).isEqualTo(new BigDecimal(10)); assertThat(schemaC.isOptional()).isEqualTo(true); assertThat(schemaC.defaultValue()).isEqualTo(null); assertThat(schemaD.isOptional()).isEqualTo(false); assertThat(schemaE.isOptional()).isEqualTo(false); assertThat(schemaE.defaultValue()).isEqualTo(BigDecimal.ZERO); assertThat(schemaF.isOptional()).isEqualTo(false); assertThat(schemaF.defaultValue()).isEqualTo(BigDecimal.ZERO); assertEmptyFieldValue(record, "G");
@Test public void unsignedMediumIntTest() throws InterruptedException { config = DATABASE.defaultConfig() .with(MySqlConnectorConfig.SNAPSHOT_MODE, MySqlConnectorConfig.SnapshotMode.INITIAL) .build(); start(MySqlConnector.class, config); // Testing.Print.enable(); SourceRecords records = consumeRecordsByTopic(EVENT_COUNT); SourceRecord record = records.recordsForTopic(DATABASE.topicForTable("UNSIGNED_MEDIUMINT_TABLE")).get(0); validate(record); Schema schemaA = record.valueSchema().fields().get(1).schema().fields().get(0).schema(); Schema schemaB = record.valueSchema().fields().get(1).schema().fields().get(1).schema(); Schema schemaC = record.valueSchema().fields().get(1).schema().fields().get(2).schema(); Schema schemaD = record.valueSchema().fields().get(1).schema().fields().get(3).schema(); Schema schemaE = record.valueSchema().fields().get(1).schema().fields().get(4).schema(); Schema schemaF = record.valueSchema().fields().get(1).schema().fields().get(5).schema(); assertThat(schemaA.isOptional()).isEqualTo(true); assertThat(schemaA.defaultValue()).isEqualTo(0); assertThat(schemaB.isOptional()).isEqualTo(true); assertThat(schemaB.defaultValue()).isEqualTo(10); assertThat(schemaC.isOptional()).isEqualTo(true); assertThat(schemaC.defaultValue()).isEqualTo(null); assertThat(schemaD.isOptional()).isEqualTo(false); assertThat(schemaE.isOptional()).isEqualTo(false); assertThat(schemaE.defaultValue()).isEqualTo(0); assertThat(schemaF.isOptional()).isEqualTo(false); assertThat(schemaF.defaultValue()).isEqualTo(0); assertEmptyFieldValue(record, "G"); }
@Test public void unsignedIntTest() throws InterruptedException { config = DATABASE.defaultConfig() .with(MySqlConnectorConfig.SNAPSHOT_MODE, MySqlConnectorConfig.SnapshotMode.INITIAL) .build(); start(MySqlConnector.class, config); // Testing.Print.enable(); SourceRecords records = consumeRecordsByTopic(EVENT_COUNT); SourceRecord record = records.recordsForTopic(DATABASE.topicForTable("UNSIGNED_INT_TABLE")).get(0); validate(record); Schema schemaA = record.valueSchema().fields().get(1).schema().fields().get(0).schema(); Schema schemaB = record.valueSchema().fields().get(1).schema().fields().get(1).schema(); Schema schemaC = record.valueSchema().fields().get(1).schema().fields().get(2).schema(); Schema schemaD = record.valueSchema().fields().get(1).schema().fields().get(3).schema(); Schema schemaE = record.valueSchema().fields().get(1).schema().fields().get(4).schema(); Schema schemaF = record.valueSchema().fields().get(1).schema().fields().get(5).schema(); assertThat(schemaA.isOptional()).isEqualTo(true); assertThat(schemaA.defaultValue()).isEqualTo(0L); assertThat(schemaB.isOptional()).isEqualTo(true); assertThat(schemaB.defaultValue()).isEqualTo(10L); assertThat(schemaC.isOptional()).isEqualTo(true); assertThat(schemaC.defaultValue()).isEqualTo(null); assertThat(schemaD.isOptional()).isEqualTo(false); assertThat(schemaE.isOptional()).isEqualTo(false); assertThat(schemaE.defaultValue()).isEqualTo(0L); assertThat(schemaF.isOptional()).isEqualTo(false); assertThat(schemaF.defaultValue()).isEqualTo(0L); assertEmptyFieldValue(record, "G"); }
@Test public void unsignedBigIntToLongTest() throws InterruptedException { config = DATABASE.defaultConfig() .with(MySqlConnectorConfig.SNAPSHOT_MODE, MySqlConnectorConfig.SnapshotMode.INITIAL) .build(); start(MySqlConnector.class, config); // Testing.Print.enable(); SourceRecords records = consumeRecordsByTopic(EVENT_COUNT); SourceRecord record = records.recordsForTopic(DATABASE.topicForTable("UNSIGNED_BIGINT_TABLE")).get(0); validate(record); Schema schemaA = record.valueSchema().fields().get(1).schema().fields().get(0).schema(); Schema schemaB = record.valueSchema().fields().get(1).schema().fields().get(1).schema(); Schema schemaC = record.valueSchema().fields().get(1).schema().fields().get(2).schema(); Schema schemaD = record.valueSchema().fields().get(1).schema().fields().get(3).schema(); Schema schemaE = record.valueSchema().fields().get(1).schema().fields().get(4).schema(); Schema schemaF = record.valueSchema().fields().get(1).schema().fields().get(5).schema(); assertThat(schemaA.isOptional()).isEqualTo(true); assertThat(schemaA.defaultValue()).isEqualTo(0L); assertThat(schemaB.isOptional()).isEqualTo(true); assertThat(schemaB.defaultValue()).isEqualTo(10L); assertThat(schemaC.isOptional()).isEqualTo(true); assertThat(schemaC.defaultValue()).isEqualTo(null); assertThat(schemaD.isOptional()).isEqualTo(false); assertThat(schemaE.isOptional()).isEqualTo(false); assertThat(schemaE.defaultValue()).isEqualTo(0L); assertThat(schemaF.isOptional()).isEqualTo(false); assertThat(schemaF.defaultValue()).isEqualTo(0L); assertEmptyFieldValue(record, "G"); }
@Test public void unsignedTinyIntTest() throws InterruptedException { config = DATABASE.defaultConfig() .with(MySqlConnectorConfig.SNAPSHOT_MODE, MySqlConnectorConfig.SnapshotMode.INITIAL) .with(MySqlConnectorConfig.DDL_PARSER_MODE, "antlr") .build(); start(MySqlConnector.class, config); // Testing.Print.enable(); SourceRecords records = consumeRecordsByTopic(EVENT_COUNT); SourceRecord record = records.recordsForTopic(DATABASE.topicForTable("UNSIGNED_TINYINT_TABLE")).get(0); validate(record); Schema schemaA = record.valueSchema().fields().get(1).schema().fields().get(0).schema(); Schema schemaB = record.valueSchema().fields().get(1).schema().fields().get(1).schema(); Schema schemaC = record.valueSchema().fields().get(1).schema().fields().get(2).schema(); Schema schemaD = record.valueSchema().fields().get(1).schema().fields().get(3).schema(); Schema schemaE = record.valueSchema().fields().get(1).schema().fields().get(4).schema(); Schema schemaF = record.valueSchema().fields().get(1).schema().fields().get(5).schema(); assertThat(schemaA.isOptional()).isEqualTo(true); assertThat(schemaA.defaultValue()).isEqualTo((short) 0); assertThat(schemaB.isOptional()).isEqualTo(true); assertThat(schemaB.defaultValue()).isEqualTo((short) 10); assertThat(schemaC.isOptional()).isEqualTo(true); assertThat(schemaC.defaultValue()).isEqualTo(null); assertThat(schemaD.isOptional()).isEqualTo(false); assertThat(schemaE.isOptional()).isEqualTo(false); assertThat(schemaE.defaultValue()).isEqualTo((short) 0); assertThat(schemaF.isOptional()).isEqualTo(false); assertThat(schemaF.defaultValue()).isEqualTo((short) 0); assertEmptyFieldValue(record, "G"); }
assertThat(schemaB.isOptional()).isEqualTo(true); assertThat(schemaB.defaultValue()).isEqualTo(null); assertThat(schemaD.isOptional()).isEqualTo(true); assertThat(schemaD.defaultValue()).isEqualTo(null); assertThat(schemaF.isOptional()).isEqualTo(true); assertThat(schemaF.defaultValue()).isEqualTo(null);
break; if (schema.isOptional()) { schemaBuilder.optional();
protected void assertField(Field field, String fieldName, Schema expectedSchema, boolean optional) { assertThat(field.name()).isEqualTo(fieldName); Schema schema = field.schema(); assertThat(schema.name()).isEqualTo(expectedSchema.name()); assertThat(schema.doc()).isEqualTo(expectedSchema.doc()); assertThat(schema.parameters()).isEqualTo(expectedSchema.parameters()); assertThat(schema.version()).isEqualTo(expectedSchema.version()); assertThat(schema.isOptional()).isEqualTo(optional); switch (expectedSchema.type()) { case STRUCT: for (Field f : expectedSchema.fields()) { assertField(schema.field(f.name()),f.name(),f.schema(),f.schema().isOptional()); } break; default: } }
void checkSchemaAndInput(Schema schema, Object input) { Preconditions.checkNotNull(schema, "schema cannot be null"); if (!schema.isOptional()) { Preconditions.checkNotNull(input, "schema is not optional so input cannot be null."); } }