private List<?> enforceFieldTypeForArray(final Schema fieldSchema, final List<?> arrayList) { final List<Object> array = new ArrayList<>(arrayList.size()); for (final Object item : arrayList) { array.add(enforceFieldType(fieldSchema.valueSchema(), item)); } return array; }
private Map<String, Object> enforceFieldTypeForMap( final Schema fieldSchema, final Map<String, ?> columnMap) { final Map<String, Object> ksqlMap = new HashMap<>(); for (final Map.Entry<String, ?> e : columnMap.entrySet()) { ksqlMap.put( enforceFieldType(Schema.OPTIONAL_STRING_SCHEMA, e.getKey()).toString(), enforceFieldType(fieldSchema.valueSchema(), e.getValue()) ); } return ksqlMap; }
private boolean compareSchemas(final Schema schema1, final Schema schema2) { if (schema1.type() != schema2.type()) { return false; } switch (schema1.type()) { case STRUCT: return compareStructSchema(schema1, schema2); case ARRAY: return compareSchemas(schema1.valueSchema(), schema2.valueSchema()); case MAP: return compareSchemas(schema1.valueSchema(), schema2.valueSchema()) && compareSchemas(schema1.keySchema(), schema2.keySchema()); default: return true; } }
@Test public void shouldGetArraySchemaFromListClass() throws NoSuchMethodException { final Type type = getClass().getDeclaredMethod("listType", List.class) .getGenericParameterTypes()[0]; final Schema schema = SchemaUtil.getSchemaFromType(type); assertThat(schema.type(), equalTo(Schema.Type.ARRAY)); assertThat(schema.valueSchema(), equalTo(Schema.OPTIONAL_FLOAT64_SCHEMA)); }
private Schema toKsqlArraySchema(final Schema schema) { return SchemaBuilder.array( toKsqlFieldSchema(schema.valueSchema()) ).optional().build(); }
@Test public void shouldGetMapSchemaFromMapClass() throws NoSuchMethodException { final Type type = getClass().getDeclaredMethod("mapType", Map.class) .getGenericParameterTypes()[0]; final Schema schema = SchemaUtil.getSchemaFromType(type); assertThat(schema.type(), equalTo(Schema.Type.MAP)); assertThat(schema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(schema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
public static String getSqlTypeName(final Schema schema) { switch (schema.type()) { case INT32: return "INT"; case INT64: return "BIGINT"; case FLOAT32: case FLOAT64: return "DOUBLE"; case BOOLEAN: return "BOOLEAN"; case STRING: return "VARCHAR"; case ARRAY: return "ARRAY<" + getSqlTypeName(schema.valueSchema()) + ">"; case MAP: return "MAP<" + getSqlTypeName(schema.keySchema()) + "," + getSqlTypeName(schema.valueSchema()) + ">"; case STRUCT: return getStructString(schema); default: throw new KsqlException(String.format("Invalid type in schema: %s.", schema.toString())); } }
public static Type getKsqlType(final Schema schema) { switch (schema.type()) { case INT32: return new PrimitiveType(Type.KsqlType.INTEGER); case INT64: return new PrimitiveType(Type.KsqlType.BIGINT); case FLOAT32: case FLOAT64: return new PrimitiveType(Type.KsqlType.DOUBLE); case BOOLEAN: return new PrimitiveType(Type.KsqlType.BOOLEAN); case STRING: return new PrimitiveType(Type.KsqlType.STRING); case ARRAY: return new Array(getKsqlType(schema.valueSchema())); case MAP: return new Map(getKsqlType(schema.valueSchema())); case STRUCT: return new Struct(getStructItems(schema)); default: throw new KsqlException(String.format("Invalid type in schema: %s.", schema.toString())); } }
@Test public void shouldGetCorrectMapSchema() throws Exception { final Schema schema = TypeUtil.getTypeSchema(new Map(new PrimitiveType(Type.KsqlType.DOUBLE))); assertThat(schema.type(), equalTo(Schema.Type.MAP)); assertThat(schema.valueSchema().type(), equalTo(Schema.Type.FLOAT64)); }
@Test public void shouldGetCorrectArraySchema() throws Exception { final Schema schema = TypeUtil.getTypeSchema(new Array(new PrimitiveType(Type.KsqlType.STRING))); assertThat(schema.type(), equalTo(Schema.Type.ARRAY)); assertThat(schema.valueSchema().type(), equalTo(Schema.Type.STRING)); }
private Schema toKsqlMapSchema(final Schema schema) { final Schema keySchema = toKsqlFieldSchema(schema.keySchema()); checkMapKeyType(keySchema.type()); return SchemaBuilder.map( Schema.OPTIONAL_STRING_SCHEMA, toKsqlFieldSchema(schema.valueSchema()) ).optional().build(); }
private static SchemaInfo buildSchemaEntity(final Schema schema) { switch (schema.type()) { case ARRAY: case MAP: return new SchemaInfo( getSchemaTypeString(schema.type()), null, buildSchemaEntity(schema.valueSchema()) ); case STRUCT: return new SchemaInfo( getSchemaTypeString(schema.type()), schema.fields() .stream() .map( f -> new FieldInfo(f.name(), buildSchemaEntity(f.schema()))) .collect(Collectors.toList()), null ); default: return new SchemaInfo(getSchemaTypeString(schema.type()), null, null); } }
@Test public void shouldTranslateStructInsideArray() { final Schema connectSchema = SchemaBuilder .struct() .field( "arrayField", SchemaBuilder.array( SchemaBuilder.struct() .field("innerIntField", Schema.OPTIONAL_INT32_SCHEMA) .build())) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("ARRAYFIELD"), notNullValue()); final Schema arraySchema = ksqlSchema.field("ARRAYFIELD").schema(); assertThat(arraySchema.type(), equalTo(Schema.Type.ARRAY)); assertThat(arraySchema.valueSchema().type(), equalTo(Schema.Type.STRUCT)); assertThat(arraySchema.valueSchema().fields().size(), equalTo(1)); assertThat(arraySchema.valueSchema().fields().get(0).name(), equalTo("INNERINTFIELD")); assertThat(arraySchema.valueSchema().fields().get(0).schema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void shouldTranslateStructInsideMap() { final Schema connectSchema = SchemaBuilder .struct() .field( "mapField", SchemaBuilder.map( Schema.STRING_SCHEMA, SchemaBuilder.struct() .field("innerIntField", Schema.INT32_SCHEMA) .build())) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.isOptional(), is(true)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema().type(), equalTo(Schema.Type.STRUCT)); assertThat(mapSchema.valueSchema().fields().size(), equalTo(1)); assertThat(mapSchema.valueSchema().fields().get(0).name(), equalTo("INNERINTFIELD")); assertThat(mapSchema.valueSchema().fields().get(0).schema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void shouldTranslateArray() { final Schema connectSchema = SchemaBuilder .struct() .field("arrayField", SchemaBuilder.array(Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("ARRAYFIELD"), notNullValue()); final Schema arraySchema = ksqlSchema.field("ARRAYFIELD").schema(); assertThat(arraySchema.type(), equalTo(Schema.Type.ARRAY)); assertThat(arraySchema.isOptional(), is(true)); assertThat(arraySchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void shouldTranslateMapWithNonStringKey() { final Schema connectSchema = SchemaBuilder .struct() .field("mapfield", SchemaBuilder.map(Schema.INT32_SCHEMA, Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void shouldTranslateMaps() { final Schema connectSchema = SchemaBuilder .struct() .field("mapField", SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
static Schema getOptionalSchema(final Schema schema) { switch (schema.type()) { case BOOLEAN: return Schema.OPTIONAL_BOOLEAN_SCHEMA; case INT32: return Schema.OPTIONAL_INT32_SCHEMA; case INT64: return Schema.OPTIONAL_INT64_SCHEMA; case FLOAT64: return Schema.OPTIONAL_FLOAT64_SCHEMA; case STRING: return Schema.OPTIONAL_STRING_SCHEMA; case ARRAY: return SchemaBuilder.array(getOptionalSchema(schema.valueSchema())).optional().build(); case MAP: return SchemaBuilder.map( getOptionalSchema(schema.keySchema()), getOptionalSchema(schema.valueSchema())) .optional().build(); case STRUCT: final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { schemaBuilder.field(field.name(), getOptionalSchema(field.schema())); } return schemaBuilder.optional().build(); default: throw new KsqlException("Unsupported type: " + schema); } }
private static org.apache.avro.Schema getAvroSchemaForField( final String namespace, final String fieldName, final Schema fieldSchema ) { switch (fieldSchema.type()) { case STRING: return unionWithNull(create(org.apache.avro.Schema.Type.STRING)); case BOOLEAN: return unionWithNull(create(org.apache.avro.Schema.Type.BOOLEAN)); case INT32: return unionWithNull(create(org.apache.avro.Schema.Type.INT)); case INT64: return unionWithNull(create(org.apache.avro.Schema.Type.LONG)); case FLOAT64: return unionWithNull(create(org.apache.avro.Schema.Type.DOUBLE)); case ARRAY: return unionWithNull(createArray( getAvroSchemaForField(namespace, fieldName, fieldSchema.valueSchema()))); case MAP: return unionWithNull(createMap( getAvroSchemaForField(namespace, fieldName, fieldSchema.valueSchema()))); case STRUCT: return unionWithNull(buildAvroSchema(namespace, fieldName, fieldSchema)); default: throw new KsqlException("Unsupported AVRO type: " + fieldSchema.type().name()); } }
@Test public void testSelectUDFs() { final String queryString = String.format( "SELECT ITEMID, " + "ORDERUNITS*10 AS Col1, " + "PRICEARRAY[0]+10 AS Col2, " + "KEYVALUEMAP['key1']*KEYVALUEMAP['key2']+10 AS Col3, " + "PRICEARRAY[1]>1000 AS Col4 " + "FROM %s " + "WHERE ORDERUNITS > 20 AND ITEMID LIKE '%%_8';", orderDataProvider.kstreamName() ); final Schema sourceSchema = orderDataProvider.schema(); final Schema resultSchema = SchemaBuilder.struct() .field("ITEMID", sourceSchema.field("ITEMID").schema()) .field("COL1", sourceSchema.field("ORDERUNITS").schema()) .field("COL2", sourceSchema.field("PRICEARRAY").schema().valueSchema()) .field("COL3", sourceSchema.field("KEYVALUEMAP").schema().valueSchema()) .field("COL4", SchemaBuilder.OPTIONAL_BOOLEAN_SCHEMA) .build(); final Map<String, GenericRow> expectedResults = new HashMap<>(); expectedResults.put("8", new GenericRow(ImmutableList.of("ITEM_8", 800.0, 1110.0, 12.0, true))); testCreateStreamAsSelect(queryString, resultSchema, expectedResults); }