@Override public Type field(Types.NestedField field, Supplier<Type> future) { Preconditions.checkArgument(sourceType.isStructType(), "Not a struct: " + sourceType); Types.StructType sourceStruct = sourceType.asStructType(); this.sourceType = sourceStruct.field(field.fieldId()).type(); try { return future.get(); } finally { sourceType = sourceStruct; } }
@Override public boolean canTransform(Type type) { return type.typeId() == TypeID.UUID; } }
@Override public Type schema(Schema schema, Type structResult) { Collection<Types.NestedField> newColumns = adds.get(TABLE_ROOT_ID); if (newColumns != null) { return addFields(structResult.asNestedType().asStructType(), newColumns); } return structResult; }
@Override public List<String> primitive(Type.PrimitiveType readPrimitive) { if (currentType.equals(readPrimitive)) { return NO_ERRORS; } if (!currentType.isPrimitiveType()) { return ImmutableList.of(String.format(": %s cannot be read as a %s", currentType.typeId().toString().toLowerCase(Locale.ENGLISH), readPrimitive)); } if (!isPromotionAllowed(currentType.asPrimitiveType(), readPrimitive)) { return ImmutableList.of(String.format(": %s cannot be promoted to %s", currentType, readPrimitive)); } // both are primitives and promotion is allowed to the read type return NO_ERRORS; } }
public static Schema fromJson(JsonNode json) { Type type = typeFromJson(json); Preconditions.checkArgument(type.isNestedType() && type.asNestedType().isStructType(), "Cannot create schema, not a struct type: %s", type); return new Schema(type.asNestedType().asStructType().fields()); }
switch (type.typeId()) { case BOOLEAN: return serdeConstants.BOOLEAN_TYPE_NAME; return String.format("decimal(%s,%s)", decimalType.precision(), decimalType.scale()); case STRUCT: final Types.StructType structType = type.asStructType(); final String nameToType = (String) structType.fields().stream().map((f) -> { return String.format("%s:%s", f.name(), fromIcebergToHiveType(f.type())); return String.format("struct<%s>", nameToType); case LIST: final Types.ListType listType = type.asListType(); return String.format("array<%s>", fromIcebergToHiveType(listType.elementType())); case MAP: final Types.MapType mapType = type.asMapType(); return String.format("map<%s,%s>", fromIcebergToHiveType(mapType.keyType()), fromIcebergToHiveType(mapType.valueType()));
Type childType = type.elementType(); for (int e = 0; e < expected.numElements(); ++e) { switch (childType.typeId()) { case BOOLEAN: case INTEGER: assertEqualsLists(prefix + ".elem " + e, childType.asListType(), expected.getArray(e), toList((Seq<?>) actual.get(e))); break; case MAP: assertEqualsMaps(prefix + ".elem " + e, childType.asMapType(), expected.getMap(e), toJavaMap((scala.collection.Map<?, ?>) actual.get(e))); break;
public static <T> T visit(com.netflix.iceberg.types.Type iType, Type type, TypeWithSchemaVisitor<T> visitor) { if (type instanceof MessageType) { Types.StructType struct = iType != null ? iType.asStructType() : null; return visitor.message(struct, (MessageType) type, visitFields(struct, type.asGroupType(), visitor)); iType.asPrimitiveType() : null; return visitor.primitive(iPrimitive, type.asPrimitiveType()); Types.NestedField element = null; if (iType != null) { list = iType.asListType(); element = list.fields().get(0); Types.NestedField valueField = null; if (iType != null) { map = iType.asMapType(); keyField = map.fields().get(0); valueField = map.fields().get(1); Types.StructType struct = iType != null ? iType.asStructType() : null; return visitor.struct(struct, group, visitFields(struct, group, visitor));
@Override public ValueReader<?> record(Schema record, List<String> names, List<ValueReader<?>> fields) { return GenericReaders.struct(AvroSchemaUtil.convert(record).asStructType(), fields); }
ColumnIdMap columnIds) { TypeDescription result; switch (type.typeId()) { case BOOLEAN: result = TypeDescription.createBoolean(); for(Types.NestedField field: type.asStructType().fields()) { result.addField(field.name(), toOrc(field.fieldId(), field.type(), columnIds)); throw new IllegalArgumentException("Unhandled type " + type.typeId());
static void toJson(Type type, JsonGenerator generator) throws IOException { if (type.isPrimitiveType()) { toJson(type.asPrimitiveType(), generator); } else { Type.NestedType nested = type.asNestedType(); switch (type.typeId()) { case STRUCT: toJson(nested.asStructType(), generator); break; case LIST: toJson(nested.asListType(), generator); break; case MAP: toJson(nested.asMapType(), generator); break; default: throw new IllegalArgumentException("Cannot write unknown type: " + type); } } }
switch (type.typeId()) { case STRUCT: Types.StructType struct = type.asNestedType().asStructType(); List<VisitFieldFuture<T>> results = Lists .newArrayListWithExpectedSize(struct.fields().size()); Types.ListType list = type.asNestedType().asListType(); return visitor.list(list, new VisitFuture<>(list.elementType(), visitor)); Types.MapType map = type.asNestedType().asMapType(); return visitor.map(map, new VisitFuture<>(map.keyType(), visitor), return visitor.primitive(type.asPrimitiveType());
@Override public Type map(Types.MapType map, Supplier<Type> keyTypeFuture, Supplier<Type> valueTypeFuture) { Preconditions.checkArgument(sourceType.isMapType(), "Not a map: " + sourceType); Types.MapType sourceMap = sourceType.asMapType(); int sourceKeyId = sourceMap.keyId(); int sourceValueId = sourceMap.valueId(); try { this.sourceType = sourceMap.keyType(); Type keyType = keyTypeFuture.get(); this.sourceType = sourceMap.valueType(); Type valueType = valueTypeFuture.get(); if (map.isValueOptional()) { return Types.MapType.ofOptional(sourceKeyId, sourceValueId, keyType, valueType); } else { return Types.MapType.ofRequired(sourceKeyId, sourceValueId, keyType, valueType); } } finally { this.sourceType = sourceMap; } }
Record l1 = GenericRecord.create(writeSchema.findType("locations").asMapType().valueType().asStructType()); l1.setField("lat", 53.992811f); l1.setField("long", -1.542616f); Record l2 = GenericRecord.create(writeSchema.findType("locations").asMapType().valueType().asStructType()); l2.setField("lat", 52.995143f); l2.setField("long", -1.539054f);
Record p1 = GenericRecord.create(writeSchema.findType("points").asListType().elementType().asStructType()); p1.setField("x", 1); p1.setField("y", 2); Record p2 = GenericRecord.create(writeSchema.findType("points").asListType().elementType().asStructType()); p2.setField("x", 3); p2.setField("y", null);
@Override public Type list(Types.ListType list, Supplier<Type> elementTypeFuture) { Preconditions.checkArgument(sourceType.isListType(), "Not a list: " + sourceType); Types.ListType sourceList = sourceType.asListType(); int sourceElementId = sourceList.elementId(); this.sourceType = sourceList.elementType(); try { if (list.isElementOptional()) { return Types.ListType.ofOptional(sourceElementId, elementTypeFuture.get()); } else { return Types.ListType.ofRequired(sourceElementId, elementTypeFuture.get()); } } finally { this.sourceType = sourceList; } }
@Override public Schema array(Schema array, Supplier<Schema> element) { if (array.getLogicalType() instanceof LogicalMap || (current.isMapType() && isKeyValueSchema(array.getElementType()))) { Preconditions.checkArgument(current.isMapType(), "Incompatible projected type: %s", current); Types.MapType m = current.asNestedType().asMapType(); this.current = Types.StructType.of(m.fields()); // create a struct to correspond to element try { Preconditions.checkArgument(current.isListType(), "Incompatible projected type: %s", current); Types.ListType list = current.asNestedType().asListType(); this.current = list.elementType(); try {
public Type field(NestedField field) { Type.Repetition repetition = field.isOptional() ? Type.Repetition.OPTIONAL : Type.Repetition.REQUIRED; int id = field.fieldId(); String name = field.name(); if (field.type().isPrimitiveType()) { return primitive(field.type().asPrimitiveType(), repetition, id, name); } else { NestedType nested = field.type().asNestedType(); if (nested.isStructType()) { return struct(nested.asStructType(), repetition, id, name); } else if (nested.isMapType()) { return map(nested.asMapType(), repetition, id, name); } else if (nested.isListType()) { return list(nested.asListType(), repetition, id, name); } throw new UnsupportedOperationException("Can't convert unknown type: " + nested); } }
@Override public List<String> map(Types.MapType readMap, Supplier<List<String>> keyErrors, Supplier<List<String>> valueErrors) { if (!currentType.isMapType()) { return ImmutableList.of(String.format(": %s cannot be read as a map", currentType)); } Types.MapType map = currentType.asNestedType().asMapType(); List<String> errors = Lists.newArrayList(); try { if (readMap.isValueRequired() && map.isValueOptional()) { errors.add(": values should be required, but are optional"); } this.currentType = map.keyType(); errors.addAll(keyErrors.get()); this.currentType = map.valueType(); errors.addAll(valueErrors.get()); return errors; } finally { this.currentType = map; } }
@Override public List<String> list(Types.ListType readList, Supplier<List<String>> elementErrors) { if (!currentType.isListType()) { return ImmutableList.of(String.format(": %s cannot be read as a list", currentType)); } Types.ListType list = currentType.asNestedType().asListType(); List<String> errors = Lists.newArrayList(); this.currentType = list.elementType(); try { if (readList.isElementRequired() && list.isElementOptional()) { errors.add(": elements should be required, but are optional"); } errors.addAll(elementErrors.get()); return errors; } finally { this.currentType = list; } }