@Override public String toString() { return String.format("table {\n%s\n}", NEWLINE.join(struct.fields().stream() .map(f -> " " + f) .collect(Collectors.toList()))); } }
@Override public Map<String, Integer> load(StructType struct) { Map<String, Integer> idToPos = Maps.newHashMap(); List<Types.NestedField> fields = struct.fields(); for (int i = 0; i < fields.size(); i += 1) { idToPos.put(fields.get(i).name(), i); } return idToPos; } });
private GenericRecord(StructType struct) { this.struct = struct; this.size = struct.fields().size(); this.values = new Object[size]; this.nameToPos = NAME_MAP_CACHE.getUnchecked(struct); }
private int find(int fieldId, Types.StructType struct) { List<Types.NestedField> fields = struct.fields(); for (int i = 0; i < fields.size(); i += 1) { if (fields.get(i).fieldId() == fieldId) { return i; } } throw new ValidationException( "Cannot find top-level field id %d in struct: %s", fieldId, struct); }
BoundReference(Types.StructType struct, int fieldId) { this.fieldId = fieldId; this.pos = find(fieldId, struct); this.type = struct.fields().get(pos).type(); }
public GroupType struct(StructType struct, Type.Repetition repetition, int id, String name) { Types.GroupBuilder<GroupType> builder = Types.buildGroup(repetition); for (NestedField field : struct.fields()) { builder.addField(field(field)); } return builder.id(id).named(name); }
public static void assertEqualsSafe(Types.StructType struct, Record rec, Row row) { List<Types.NestedField> fields = struct.fields(); for (int i = 0; i < fields.size(); i += 1) { Type fieldType = fields.get(i).type(); Object expectedValue = rec.get(i); Object actualValue = row.get(i); assertEqualsSafe(fieldType, expectedValue, actualValue); } }
@Test public void testWriteWithCustomDataLocation() throws IOException { File location = createTableFolder(); File tablePropertyDataLocation = temp.newFolder("test-table-property-data-dir"); Table table = createTable(new Schema(SUPPORTED_PRIMITIVES.fields()), location); table.updateProperties().set( TableProperties.WRITE_NEW_DATA_LOCATION, tablePropertyDataLocation.getAbsolutePath()).commit(); writeAndValidateWithLocations(table, location, tablePropertyDataLocation); }
/** * @return a List of the {@link Types.NestedField columns} in this Schema. */ public List<Types.NestedField> columns() { return struct.fields(); }
static void assertEquals(Types.StructType struct, Record expected, Record actual) { List<Types.NestedField> fields = struct.fields(); for (int i = 0; i < fields.size(); i += 1) { Type fieldType = fields.get(i).type(); Object expectedValue = expected.get(i); Object actualValue = actual.get(i); assertEquals(fieldType, expectedValue, actualValue); } }
public static void assertEqualsUnsafe(Types.StructType struct, Record rec, InternalRow row) { List<Types.NestedField> fields = struct.fields(); for (int i = 0; i < fields.size(); i += 1) { Type fieldType = fields.get(i).type(); Object expectedValue = rec.get(i); Object actualValue = row.get(i, convert(fieldType)); assertEqualsUnsafe(fieldType, expectedValue, actualValue); } }
@Override public DataType struct(Types.StructType struct, List<DataType> fieldResults) { List<Types.NestedField> fields = struct.fields(); List<StructField> sparkFields = Lists.newArrayListWithExpectedSize(fieldResults.size()); for (int i = 0; i < fields.size(); i += 1) { Types.NestedField field = fields.get(i); DataType type = fieldResults.get(i); sparkFields.add(StructField.apply(field.name(), type, field.isOptional(), Metadata.empty())); } return StructType$.MODULE$.apply(sparkFields); }
public static void assertEquals(Types.StructType struct, Record expected, Record actual) { List<Types.NestedField> fields = struct.fields(); for (int i = 0; i < fields.size(); i += 1) { Type fieldType = fields.get(i).type(); Object expectedValue = expected.get(i); Object actualValue = actual.get(i); assertEquals(fieldType, expectedValue, actualValue); } }
private static Record projectFlat(Schema projection, Record record) { org.apache.avro.Schema avroSchema = AvroSchemaUtil.convert(projection, "test"); Record result = new Record(avroSchema); List<Types.NestedField> fields = projection.asStruct().fields(); for (int i = 0; i < fields.size(); i += 1) { Types.NestedField field = fields.get(i); result.put(i, record.get(field.name())); } return result; }
/** * Used by Avro reflection to instantiate this class when reading manifest files. */ public PartitionData(Schema schema) { this.partitionType = AvroSchemaUtil.convert(schema).asNestedType().asStructType(); this.size = partitionType.fields().size(); this.data = new Object[size]; this.stringSchema = schema.toString(); this.schema = schema; }
private static Types.StructType addFields(Types.StructType struct, Collection<Types.NestedField> adds) { List<Types.NestedField> newFields = Lists.newArrayList(struct.fields()); newFields.addAll(adds); return Types.StructType.of(newFields); } }
private static Schema applyChanges(Schema schema, List<Integer> deletes, Map<Integer, Types.NestedField> updates, Multimap<Integer, Types.NestedField> adds) { Types.StructType struct = TypeUtil .visit(schema, new ApplyChanges(deletes, updates, adds)) .asNestedType().asStructType(); return new Schema(struct.fields()); }
@Test public void testSimpleStruct() throws IOException { writeAndValidate(new Schema(SUPPORTED_PRIMITIVES.fields())); }
public static Schema fromJson(JsonNode json) { Type type = typeFromJson(json); Preconditions.checkArgument(type.isNestedType() && type.asNestedType().isStructType(), "Cannot create schema, not a struct type: %s", type); return new Schema(type.asNestedType().asStructType().fields()); }
/** * Convert an ORC schema to an Iceberg schema. * @param schema the ORC schema * @param columnIds the column ids * @return the Iceberg schema */ public Schema fromOrc(TypeDescription schema, ColumnIdMap columnIds) { return new Schema(convertOrcToType(schema, columnIds).asStructType().fields()); }