@Override public Type list(Types.ListType list, Supplier<Type> future) { int newId = nextId.get(); if (list.isElementOptional()) { return Types.ListType.ofOptional(newId, future.get()); } else { return Types.ListType.ofRequired(newId, future.get()); } }
@Override public Type list(Types.ListType list, Supplier<Type> elementTypeFuture) { Preconditions.checkArgument(sourceType.isListType(), "Not a list: " + sourceType); Types.ListType sourceList = sourceType.asListType(); int sourceElementId = sourceList.elementId(); this.sourceType = sourceList.elementType(); try { if (list.isElementOptional()) { return Types.ListType.ofOptional(sourceElementId, elementTypeFuture.get()); } else { return Types.ListType.ofRequired(sourceElementId, elementTypeFuture.get()); } } finally { this.sourceType = sourceList; } }
@Override public Type list(Types.ListType list, Type result) { // use field to apply updates Type elementResult = field(list.fields().get(0), result); if (elementResult == null) { throw new IllegalArgumentException("Cannot delete element type from list: " + list); } if (list.elementType() == elementResult) { return list; } if (list.isElementOptional()) { return Types.ListType.ofOptional(list.elementId(), elementResult); } else { return Types.ListType.ofRequired(list.elementId(), elementResult); } }
@Override public Type list(GroupType array, Type elementType) { GroupType repeated = array.getType(0).asGroupType(); org.apache.parquet.schema.Type element = repeated.getType(0); Preconditions.checkArgument( !element.isRepetition(Repetition.REPEATED), "Elements cannot have repetition REPEATED: {}", element); int elementFieldId = getId(element); addAlias(element.getName(), elementFieldId); if (element.isRepetition(Repetition.OPTIONAL)) { return Types.ListType.ofOptional(elementFieldId, elementType); } else { return Types.ListType.ofRequired(elementFieldId, elementType); } }
@Test public void testTupleInMap() throws IOException { Schema icebergSchema = new Schema( optional( 1, "nested_list", MapType.ofOptional( 2, 3, StringType.get(), ListType.ofOptional( 4, StructType.of( required(5, "id", LongType.get()), optional(6, "data", StringType.get())))))); ResourceSchema pigSchema = SchemaUtil.convert(icebergSchema); assertEquals("nested_list:[{(id:long,data:chararray)}]", pigSchema.toString()); // The output should contain a nested struct within a list within a map, I think. }
@Override public Type list(Types.ListType list, Type elementResult) { if (selected.contains(list.elementId())) { return list; } else if (elementResult != null) { if (list.elementType() == elementResult) { return list; } else if (list.isElementOptional()) { return Types.ListType.ofOptional(list.elementId(), elementResult); } else { return Types.ListType.ofRequired(list.elementId(), elementResult); } } return null; }
@Test public void testLists() throws Exception { Type[] maps = new Type[] { Types.ListType.ofOptional(2, Types.DoubleType.get()), Types.ListType.ofRequired(5, Types.DoubleType.get()) }; for (Type list : maps) { Type copy = TestHelpers.roundTripSerialize(list); Assert.assertEquals("List serialization should be equal to starting type", list, copy); Assert.assertSame("List serialization should preserve identity type", Types.DoubleType.get(), list.asNestedType().asListType().elementType()); } }
@Test public void testArray() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", ListType.ofOptional(2, Types.StringType.get()))); writeAndValidate(schema); }
private static Types.ListType listFromJson(JsonNode json) { int elementId = JsonUtil.getInt(ELEMENT_ID, json); Type elementType = typeFromJson(json.get(ELEMENT)); boolean isRequired = JsonUtil.getBool(ELEMENT_REQUIRED, json); if (isRequired) { return Types.ListType.ofRequired(elementId, elementType); } else { return Types.ListType.ofOptional(elementId, elementType); } }
@Test public void testArrayOfStructs() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", ListType.ofOptional(2, SUPPORTED_PRIMITIVES))); writeAndValidate(schema); }
@Test public void testRequiredListElement() { Schema write = new Schema(required(0, "list_field", Types.ListType.ofOptional( 1, Types.IntegerType.get() ))); Schema read = new Schema(required(0, "list_field", Types.ListType.ofRequired( 1, Types.IntegerType.get() ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain that elements are optional", errors.get(0).contains("elements should be required, but are optional")); }
@Test public void testIncompatibleListElement() { Schema write = new Schema(required(0, "list_field", Types.ListType.ofOptional( 1, Types.IntegerType.get() ))); Schema read = new Schema(required(0, "list_field", Types.ListType.ofOptional( 1, Types.StringType.get() ))); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("cannot be promoted to string")); }
@Test public void nestedBags() throws IOException { convertToPigSchema(new Schema( optional(1, "nested", ListType.ofOptional(2, ListType.ofOptional(3, ListType.ofOptional(4, DoubleType.get())))) ), "nested:{({({(double)})})}", ""); }
@Test public void testArrayOfStructs() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", ListType.ofOptional(2, SUPPORTED_PRIMITIVES))); writeAndValidate(schema); }
@Test public void testArray() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", ListType.ofOptional(2, Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testIncompatibleListAndPrimitive() { Schema write = new Schema(required(0, "list_field", Types.ListType.ofOptional( 1, Types.IntegerType.get() ))); Schema read = new Schema(required(0, "list_field", Types.StringType.get())); List<String> errors = CheckCompatibility.writeCompatibilityErrors(read, write); Assert.assertEquals("Should produce 1 error message", 1, errors.size()); Assert.assertTrue("Should complain about incompatible types", errors.get(0).contains("list cannot be read as a string")); }
@Test public void testArrayOfStructs() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", ListType.ofOptional(2, SUPPORTED_PRIMITIVES))); writeAndValidate(schema); }
@Test public void testArray() throws IOException { Schema schema = new Schema( required(0, "id", LongType.get()), optional(1, "data", ListType.ofOptional(2, Types.StringType.get()))); writeAndValidate(schema); }
@Test public void testComplex() throws IOException { convertToPigSchema( new Schema( optional(1, "bag", ListType.ofOptional(2, BooleanType.get())), optional(3, "map", MapType.ofOptional(4,5, StringType.get(), DoubleType.get())), optional(6, "tuple", StructType.of(optional(7, "i", IntegerType.get()), optional(8,"f", FloatType.get()))) ),"bag:{(boolean)},map:[double],tuple:(i:int,f:float)", null ); }
@Override public Type array(ArrayType array, Type elementType) { if (array.containsNull()) { return Types.ListType.ofOptional(getNextId(), elementType); } else { return Types.ListType.ofRequired(getNextId(), elementType); } }