@Test public void shouldGetMapSchemaFromMapClass() throws NoSuchMethodException { final Type type = getClass().getDeclaredMethod("mapType", Map.class) .getGenericParameterTypes()[0]; final Schema schema = SchemaUtil.getSchemaFromType(type); assertThat(schema.type(), equalTo(Schema.Type.MAP)); assertThat(schema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(schema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
((Map<Object, Object>) object).forEach( (key, value) -> ksqlMap.put( replaceSchema(schema.keySchema(), key), replaceSchema(schema.valueSchema(), value) ));
return map.entrySet().stream() .collect(Collectors.toMap( k -> getOptionalValue(schema.keySchema(), k), v -> getOptionalValue(schema.valueSchema(), v) ));
private boolean compareSchemas(final Schema schema1, final Schema schema2) { if (schema1.type() != schema2.type()) { return false; } switch (schema1.type()) { case STRUCT: return compareStructSchema(schema1, schema2); case ARRAY: return compareSchemas(schema1.valueSchema(), schema2.valueSchema()); case MAP: return compareSchemas(schema1.valueSchema(), schema2.valueSchema()) && compareSchemas(schema1.keySchema(), schema2.keySchema()); default: return true; } }
public static String getSqlTypeName(final Schema schema) { switch (schema.type()) { case INT32: return "INT"; case INT64: return "BIGINT"; case FLOAT32: case FLOAT64: return "DOUBLE"; case BOOLEAN: return "BOOLEAN"; case STRING: return "VARCHAR"; case ARRAY: return "ARRAY<" + getSqlTypeName(schema.valueSchema()) + ">"; case MAP: return "MAP<" + getSqlTypeName(schema.keySchema()) + "," + getSqlTypeName(schema.valueSchema()) + ">"; case STRUCT: return getStructString(schema); default: throw new KsqlException(String.format("Invalid type in schema: %s.", schema.toString())); } }
private Schema toKsqlMapSchema(final Schema schema) { final Schema keySchema = toKsqlFieldSchema(schema.keySchema()); checkMapKeyType(keySchema.type()); return SchemaBuilder.map( Schema.OPTIONAL_STRING_SCHEMA, toKsqlFieldSchema(schema.valueSchema()) ).optional().build(); }
break; case MAP: appendAdditional("key", schema.keySchema()); appendAdditional("value", schema.valueSchema()); break;
@Test public void shouldTranslateMaps() { final Schema connectSchema = SchemaBuilder .struct() .field("mapField", SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void shouldTranslateMapWithNonStringKey() { final Schema connectSchema = SchemaBuilder .struct() .field("mapfield", SchemaBuilder.map(Schema.INT32_SCHEMA, Schema.INT32_SCHEMA)) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
@Test public void shouldTranslateStructInsideMap() { final Schema connectSchema = SchemaBuilder .struct() .field( "mapField", SchemaBuilder.map( Schema.STRING_SCHEMA, SchemaBuilder.struct() .field("innerIntField", Schema.INT32_SCHEMA) .build())) .build(); final Schema ksqlSchema = schemaTranslator.toKsqlSchema(connectSchema); assertThat(ksqlSchema.field("MAPFIELD"), notNullValue()); final Schema mapSchema = ksqlSchema.field("MAPFIELD").schema(); assertThat(mapSchema.type(), equalTo(Schema.Type.MAP)); assertThat(mapSchema.isOptional(), is(true)); assertThat(mapSchema.keySchema(), equalTo(Schema.OPTIONAL_STRING_SCHEMA)); assertThat(mapSchema.valueSchema().type(), equalTo(Schema.Type.STRUCT)); assertThat(mapSchema.valueSchema().fields().size(), equalTo(1)); assertThat(mapSchema.valueSchema().fields().get(0).name(), equalTo("INNERINTFIELD")); assertThat(mapSchema.valueSchema().fields().get(0).schema(), equalTo(Schema.OPTIONAL_INT32_SCHEMA)); }
static Schema getOptionalSchema(final Schema schema) { switch (schema.type()) { case BOOLEAN: return Schema.OPTIONAL_BOOLEAN_SCHEMA; case INT32: return Schema.OPTIONAL_INT32_SCHEMA; case INT64: return Schema.OPTIONAL_INT64_SCHEMA; case FLOAT64: return Schema.OPTIONAL_FLOAT64_SCHEMA; case STRING: return Schema.OPTIONAL_STRING_SCHEMA; case ARRAY: return SchemaBuilder.array(getOptionalSchema(schema.valueSchema())).optional().build(); case MAP: return SchemaBuilder.map( getOptionalSchema(schema.keySchema()), getOptionalSchema(schema.valueSchema())) .optional().build(); case STRUCT: final SchemaBuilder schemaBuilder = SchemaBuilder.struct(); for (final Field field : schema.fields()) { schemaBuilder.field(field.name(), getOptionalSchema(field.schema())); } return schemaBuilder.optional().build(); default: throw new KsqlException("Unsupported type: " + schema); } }
case MAP: schemaBuilder = SchemaBuilder.map( buildAvroCompatibleSchema(schema.keySchema(), typeNameGenerator.with(TypeNameGenerator.MAP_KEY_NAME)), buildAvroCompatibleSchema(schema.valueSchema(),
private static Object projectMap(Schema source, Object record, Schema target) throws SchemaProjectorException { Map<?, ?> map = (Map<?, ?>) record; Map<Object, Object> retMap = new HashMap<>(); for (Map.Entry<?, ?> entry : map.entrySet()) { Object key = entry.getKey(); Object value = entry.getValue(); Object retKey = project(source.keySchema(), key, target.keySchema()); Object retValue = project(source.valueSchema(), value, target.valueSchema()); retMap.put(retKey, retValue); } return retMap; }
protected static Map<Object, Object> alignMapKeysAndValuesWithSchema(Schema mapSchema, Map<Object, Object> input) { if (mapSchema == null) { return input; } Schema keySchema = mapSchema.keySchema(); Schema valueSchema = mapSchema.valueSchema(); Map<Object, Object> result = new LinkedHashMap<>(); for (Map.Entry<?, ?> entry : input.entrySet()) { Object newKey = convertTo(keySchema, null, entry.getKey()); Object newValue = convertTo(valueSchema, null, entry.getValue()); result.put(newKey, newValue); } return result; }
public static TypeInfo convertMap(Schema schema) { return TypeInfoFactory.getMapTypeInfo( convert(schema.keySchema()), convert(schema.valueSchema())); }
private com.google.cloud.bigquery.Field.Builder convertMap(Schema kafkaConnectSchema, String fieldName) { Schema keySchema = kafkaConnectSchema.keySchema(); Schema valueSchema = kafkaConnectSchema.valueSchema(); com.google.cloud.bigquery.Field keyField = convertField(keySchema, MAP_KEY_FIELD_NAME).build(); com.google.cloud.bigquery.Field valueField = convertField(valueSchema, MAP_VALUE_FIELD_NAME).build(); com.google.cloud.bigquery.Field.Builder bigQueryRecordBuilder = com.google.cloud.bigquery.Field.newBuilder(fieldName, LegacySQLTypeName.RECORD, keyField, valueField); return bigQueryRecordBuilder.setMode(com.google.cloud.bigquery.Field.Mode.REPEATED); }
Storage(Schema schema) { this.name = schema.name(); this.doc = schema.doc(); this.type = schema.type(); this.defaultValue = schema.defaultValue(); this.version = schema.version(); this.parameters = schema.parameters(); this.isOptional = schema.isOptional(); if (Schema.Type.MAP == this.type) { this.keySchema = schema.keySchema(); this.valueSchema = schema.valueSchema(); } else if (Schema.Type.ARRAY == this.type) { this.keySchema = null; this.valueSchema = schema.valueSchema(); } else if (Schema.Type.STRUCT == this.type) { this.fieldSchemas = new LinkedHashMap<>(); for (Field field : schema.fields()) { this.fieldSchemas.put(field.name(), field.schema()); } } }
@Override public R apply(R record) { final Schema schema = operatingSchema(record); requireSchema(schema, "updating schema metadata"); final boolean isArray = schema.type() == Schema.Type.ARRAY; final boolean isMap = schema.type() == Schema.Type.MAP; final Schema updatedSchema = new ConnectSchema( schema.type(), schema.isOptional(), schema.defaultValue(), schemaName != null ? schemaName : schema.name(), schemaVersion != null ? schemaVersion : schema.version(), schema.doc(), schema.parameters(), schema.fields(), isMap ? schema.keySchema() : null, isMap || isArray ? schema.valueSchema() : null ); return newRecord(record, updatedSchema); }