public Builder addMapField(String name, FieldType keyType, FieldType valueType) { fields.add(Field.of(name, FieldType.map(keyType, valueType))); return this; }
return Schema.FieldType.map( Schema.FieldType.STRING, toFieldType(avroSchema.getValueType()));
@Test public void testCreateMapWithNullValue() { Map<Integer, String> data = new HashMap(); data.put(1, "value1"); data.put(2, "value2"); data.put(3, null); data.put(4, null); Schema type = Stream.of(Schema.Field.of("map", FieldType.map(FieldType.INT32, FieldType.STRING, true))) .collect(toSchema()); Row row = Row.withSchema(type).addValue(data).build(); assertEquals(data, row.getMap("map")); }
Schema.Field.of("f_boolean", FieldType.BOOLEAN).withNullable(true), Schema.Field.of("f_array", FieldType.array(FieldType.DATETIME)).withNullable(true), Schema.Field.of("f_map", FieldType.map(FieldType.INT32, FieldType.DOUBLE)) .withNullable(true)) .collect(toSchema());
@Test public void testExecute_createTableWithPrefixMapField() throws Exception { InMemoryMetaStore metaStore = new InMemoryMetaStore(); metaStore.registerProvider(new TextTableProvider()); BeamSqlCli cli = new BeamSqlCli().metaStore(metaStore); cli.execute( "CREATE EXTERNAL TABLE person (\n" + "id int COMMENT 'id', \n" + "name varchar COMMENT 'name', \n" + "age int COMMENT 'age', \n" + "tags MAP<VARCHAR, VARCHAR>, \n" + "nestedMap MAP<INTEGER, MAP<VARCHAR, INTEGER>> \n" + ") \n" + "TYPE 'text' \n" + "COMMENT '' LOCATION '/home/admin/orders'"); Table table = metaStore.getTables().get("person"); assertNotNull(table); assertEquals( Stream.of( Field.of("id", INTEGER).withDescription("id").withNullable(true), Field.of("name", VARCHAR).withDescription("name").withNullable(true), Field.of("age", INTEGER).withDescription("age").withNullable(true), Field.of("tags", Schema.FieldType.map(VARCHAR, VARCHAR)).withNullable(true), Field.of( "nestedMap", Schema.FieldType.map(INTEGER, Schema.FieldType.map(VARCHAR, INTEGER))) .withNullable(true)) .collect(toSchema()), table.getSchema()); }
@Test public void testSelectAll() { PCollection<Row> input = pCollectionOf2Elements(); Schema resultType = Schema.builder() .addInt32Field("f_int") .addNullableField( "f_map", Schema.FieldType.map(Schema.FieldType.STRING, Schema.FieldType.INT32)) .build(); PCollection<Row> result = input.apply( "sqlQuery", SqlTransform.query("SELECT f_int, f_intStringMap as f_map FROM PCOLLECTION")); PAssert.that(result) .containsInAnyOrder( Row.withSchema(resultType) .addValues(1, ImmutableMap.of("key11", 11, "key22", 22)) .build(), Row.withSchema(resultType) .addValues(2, ImmutableMap.of("key33", 33, "key44", 44, "key55", 55)) .build()); pipeline.run(); }
@Test public void testCreateMapWithMapValueWithNull() { Map<Integer, Map<Integer, String>> data = new HashMap(); Map<Integer, String> innerData = new HashMap(); innerData.put(11, null); innerData.put(12, "value3"); data.put(1, ImmutableMap.of(1, "value1")); data.put(2, ImmutableMap.of(2, "value2")); data.put(3, null); data.put(4, innerData); Schema type = Stream.of( Schema.Field.of( "map", FieldType.map( FieldType.INT32, FieldType.map(FieldType.INT32, FieldType.STRING, true), true))) .collect(toSchema()); Row row = Row.withSchema(type).addValue(data).build(); assertEquals(data, row.getMap("map")); }
@Test public void testCreateMapWithMapValue() { Map<Integer, Map<Integer, String>> data = ImmutableMap.<Integer, Map<Integer, String>>builder() .put(1, ImmutableMap.of(1, "value1")) .put(2, ImmutableMap.of(2, "value2")) .build(); Schema type = Stream.of( Schema.Field.of( "map", FieldType.map( FieldType.INT32, FieldType.map(FieldType.INT32, FieldType.STRING)))) .collect(toSchema()); Row row = Row.withSchema(type).addValue(data).build(); assertEquals(data, row.getMap("map")); }
private void validatePubsubMessageSchema(Table tableDefinition) { Schema schema = tableDefinition.getSchema(); if (schema.getFieldCount() != 3 || !fieldPresent(schema, TIMESTAMP_FIELD, TIMESTAMP) || !fieldPresent(schema, ATTRIBUTES_FIELD, Schema.FieldType.map(VARCHAR, VARCHAR)) || !(schema.hasField(PAYLOAD_FIELD) && ROW.equals(schema.getField(PAYLOAD_FIELD).getType().getTypeName()))) { throw new IllegalArgumentException( "Unsupported schema specified for Pubsub source in CREATE TABLE. " + "CREATE TABLE for Pubsub topic should define exactly the following fields: " + "'event_timestamp' field of type 'TIMESTAMP', 'attributes' field of type " + "MAP<VARCHAR, VARCHAR>, and 'payload' field of type 'ROW<...>' which matches the " + "payload JSON format."); } }
@Test public void testCreatesArrayOfMap() { List<Map<Integer, String>> data = ImmutableList.<Map<Integer, String>>builder() .add(ImmutableMap.of(1, "value1")) .add(ImmutableMap.of(2, "value2")) .build(); Schema type = Stream.of( Schema.Field.of( "array", FieldType.array(FieldType.map(FieldType.INT32, FieldType.STRING)))) .collect(toSchema()); Row row = Row.withSchema(type).addArray(data).build(); assertEquals(data, row.getArray("array")); }
@Test public void testCreateMapWithArrayValue() { Map<Integer, List<String>> data = ImmutableMap.<Integer, List<String>>builder() .put(1, Arrays.asList("value1")) .put(2, Arrays.asList("value2")) .build(); Schema type = Stream.of( Schema.Field.of( "map", FieldType.map(FieldType.INT32, FieldType.array(FieldType.STRING)))) .collect(toSchema()); Row row = Row.withSchema(type).addValue(data).build(); assertEquals(data, row.getMap("map")); }
@Test public void testCreateMapWithPrimitiveValue() { Map<Integer, String> data = ImmutableMap.<Integer, String>builder() .put(1, "value1") .put(2, "value2") .put(3, "value3") .put(4, "value4") .build(); Schema type = Stream.of(Schema.Field.of("map", FieldType.map(FieldType.INT32, FieldType.STRING))) .collect(toSchema()); Row row = Row.withSchema(type).addValue(data).build(); assertEquals(data, row.getMap("map")); }
@Test public void testCreateMapWithRowValue() { Schema nestedType = Stream.of(Schema.Field.of("f1_str", FieldType.STRING)).collect(toSchema()); Map<Integer, Row> data = ImmutableMap.<Integer, Row>builder() .put(1, Row.withSchema(nestedType).addValues("one").build()) .put(2, Row.withSchema(nestedType).addValues("two").build()) .build(); Schema type = Stream.of(Schema.Field.of("map", FieldType.map(FieldType.INT32, FieldType.row(nestedType)))) .collect(toSchema()); Row row = Row.withSchema(type).addValue(data).build(); assertEquals(data, row.getMap("map")); }
@Test public void testMapTypeToFieldType() { assertEquals( FieldType.map(FieldType.STRING, FieldType.INT64), FieldTypeDescriptors.fieldTypeForJavaType( TypeDescriptors.maps(TypeDescriptors.strings(), TypeDescriptors.longs()))); assertEquals( FieldType.map(FieldType.STRING, FieldType.array(FieldType.INT64)), FieldTypeDescriptors.fieldTypeForJavaType( TypeDescriptors.maps( TypeDescriptors.strings(), TypeDescriptors.lists(TypeDescriptors.longs())))); } }
public static FieldType toFieldType(RelDataType calciteType) { switch (calciteType.getSqlTypeName()) { case ARRAY: case MULTISET: return FieldType.array(toFieldType(calciteType.getComponentType())); case MAP: return FieldType.map( toFieldType(calciteType.getKeyType()), toFieldType(calciteType.getValueType())); case ROW: return FieldType.row(toSchema(calciteType)); default: return toFieldType(calciteType.getSqlTypeName()); } }
@Override public FieldType generateFieldType(SourceOfRandomness random, GenerationStatus status) { return FieldType.map( PRIMITIVE_TYPES.generate(random, status), ANY_TYPE.generate(random, status)); } }
@Test @Ignore public void testConsistentWithEqualsMapWithBytesKeyField() throws Exception { FieldType fieldType = FieldType.map(FieldType.BYTES, FieldType.INT32); Schema schema = Schema.of(Schema.Field.of("f1", fieldType)); RowCoder coder = RowCoder.of(schema); Map<byte[], Integer> map1 = Collections.singletonMap(new byte[] {1, 2, 3, 4}, 1); Row row1 = Row.withSchema(schema).addValue(map1).build(); Map<byte[], Integer> map2 = Collections.singletonMap(new byte[] {1, 2, 3, 4}, 1); Row row2 = Row.withSchema(schema).addValue(map2).build(); Assume.assumeTrue(coder.consistentWithEquals()); CoderProperties.coderConsistentWithEquals(coder, row1, row2); }
@Test public void testCastMap() { Object output = Cast.castValue( ImmutableMap.of((short) 1, 1, (short) 2, 2, (short) 3, 3), Schema.FieldType.map(Schema.FieldType.INT16, Schema.FieldType.INT32), Schema.FieldType.map(Schema.FieldType.INT32, Schema.FieldType.INT64)); assertEquals(ImmutableMap.of(1, 1L, 2, 2L, 3, 3L), output); }
private static FieldType getMapFieldType(TypeDescriptor typeDescriptor) { TypeDescriptor<Collection<?>> map = typeDescriptor.getSupertype(Map.class); if (map.getType() instanceof ParameterizedType) { ParameterizedType ptype = (ParameterizedType) map.getType(); java.lang.reflect.Type[] params = ptype.getActualTypeArguments(); return FieldType.map( fieldTypeForJavaType(TypeDescriptor.of(params[0])), fieldTypeForJavaType(TypeDescriptor.of(params[1]))); } throw new RuntimeException("Cound not determine array parameter type for field."); } }
@Test public void testMapTypeToJavaType() { assertEquals( TypeDescriptors.maps(TypeDescriptors.strings(), TypeDescriptors.longs()), FieldTypeDescriptors.javaTypeForFieldType( FieldType.map(FieldType.STRING, FieldType.INT64))); assertEquals( TypeDescriptors.maps( TypeDescriptors.strings(), TypeDescriptors.lists(TypeDescriptors.longs())), FieldTypeDescriptors.javaTypeForFieldType( FieldType.map(FieldType.STRING, FieldType.array(FieldType.INT64)))); }