public static MapType ofOptional(int keyId, int valueId, Type keyType, Type valueType) { Preconditions.checkNotNull(valueType, "Value type cannot be null"); return new MapType( NestedField.required(keyId, "key", keyType), NestedField.optional(valueId, "value", valueType)); }
@Test public void testRequiredSchemaField() { Schema write = new Schema(optional(1, "from_field", Types.IntegerType.get())); Schema read = new Schema(required(1, "to_field", Types.IntegerType.get())); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain that a required column is optional", errors.get(0).contains("should be required, but is optional")); }
@Test public void testMissingSchemaField() { Schema write = new Schema(required(0, "other_field", Types.IntegerType.get())); Schema read = new Schema(required(1, "to_field", Types.IntegerType.get())); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain that a required column is missing", errors.get(0).contains("is required, but is missing")); }
@Test public void testIncompatibleListAndPrimitive() { Schema write = new Schema(required(0, "list_field", Types.ListType.ofOptional( 1, Types.IntegerType.get() ))); Schema read = new Schema(required(0, "list_field", Types.StringType.get())); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("list cannot be read as a string")); }
@Test public void testMapOfStructs() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), SUPPORTED_PRIMITIVES))); writeAndValidate(schema); }
@Test public void testMap() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testCharSeqValue() { StructType struct = StructType.of(required(34, "s", Types.StringType.get())); Evaluator evaluator = new Evaluator(struct, equal("s", "abc")); Assert.assertTrue("string(abc) == utf8(abc) => true", evaluator.eval(TestHelpers.Row.of(new Utf8("abc")))); Assert.assertFalse("string(abc) == utf8(abcd) => false", evaluator.eval(TestHelpers.Row.of(new Utf8("abcd")))); } }
@Test public void testIncompatibleMapAndPrimitive() { Schema write = new Schema(required(0, "map_field", Types.MapType.ofOptional( 1, 2, Types.StringType.get(), Types.IntegerType.get() ))); Schema read = new Schema(required(0, "map_field", Types.StringType.get())); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("map cannot be read as a string")); }
@Test public void testArray() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", ListType.ofOptional(2, Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testComplexMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StructType.of( required(4, "i", Types.IntegerType.get()), optional(5, "s", Types.StringType.get())), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testMap() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testArrayOfStructs() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", ListType.ofOptional(2, SUPPORTED_PRIMITIVES))); writeAndValidate(schema); }
@Test public void testMapOfStructs() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), SUPPORTED_PRIMITIVES))); writeAndValidate(schema); }
@Test public void testComplexMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StructType.of( required(4, "i", Types.IntegerType.get()), optional(5, "s", Types.StringType.get())), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testMapOfStructs() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), SUPPORTED_PRIMITIVES))); writeAndValidate(schema); }
@Test public void testIncompatibleStructAndPrimitive() { Schema write = new Schema(required(0, "nested", Types.StructType.of( required(1, "from_field", Types.StringType.get()) ))); Schema read = new Schema(required(0, "nested", Types.StringType.get())); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("struct cannot be read as a string")); }
@Test public void testMap() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, Types.StringType.get(), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testComplexMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, StructType.of( required(4, "i", Types.IntegerType.get()), optional(5, "s", Types.StringType.get())), Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testNumericMapKey() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", MapType.ofOptional(2, 3, LongType.get(), Types.StringType.get()))); writeAndValidate(schema); }