@Override public T get() { return visit(type, visitor); } }
/** * Convert a {@link Type} to a {@link DataType Spark type}. * * @param type a Type * @return the equivalent Spark type * @throws IllegalArgumentException if the type cannot be converted to Spark */ public static DataType convert(Type type) { return visit(type, new TypeToSparkType()); }
/** * Assigns fresh ids from the {@link NextID nextId function} for all fields in a type. * * @param type a type * @param nextId an id assignment function * @return an structurally identical type with new ids assigned by the nextId function */ public static Type assignFreshIds(Type type, NextID nextId) { return TypeUtil.visit(type, new AssignFreshIds(nextId)); }
/** * Returns a list of compatibility errors for reading with the given read schema. * * @param readSchema a read schema * @param writeSchema a write schema * @return a list of error details, or an empty list if there are no compatibility problems */ public static List<String> readCompatibilityErrors(Schema readSchema, Schema writeSchema) { return TypeUtil.visit(readSchema, new CheckCompatibility(writeSchema, false)); }
/** * Convert a {@link Schema} to a {@link DataType Spark type}. * * @param schema a Schema * @return the equivalent Spark type * @throws IllegalArgumentException if the type cannot be converted to Spark */ public static StructType convert(Schema schema) { return (StructType) visit(schema, new TypeToSparkType()); }
/** * Returns a list of compatibility errors for writing with the given write schema. * * @param readSchema a read schema * @param writeSchema a write schema * @return a list of error details, or an empty list if there are no compatibility problems */ public static List<String> writeCompatibilityErrors(Schema readSchema, Schema writeSchema) { return TypeUtil.visit(readSchema, new CheckCompatibility(writeSchema, true)); }
@Override public InternalRow next() { rowsLeft -= 1; return (InternalRow) TypeUtil.visit(schema, generator); } };
public static Schema convert(com.netflix.iceberg.Schema schema, Map<Types.StructType, String> names) { return TypeUtil.visit(schema, new TypeToSchema(names)); }
public static Map<String, Integer> indexByName(Types.StructType struct) { return visit(struct, new IndexByName()); }
public static Set<Integer> getProjectedIds(Schema schema) { return visit(schema, new GetProjectedIds()); }
public static Map<Integer, Types.NestedField> indexById(Types.StructType struct) { return visit(struct, new IndexById()); }
public static Set<Integer> getProjectedIds(Type schema) { if (schema.isPrimitiveType()) { return ImmutableSet.of(); } return ImmutableSet.copyOf(visit(schema, new GetProjectedIds())); }
public static Map<Type, Schema> convertTypes(Types.StructType type, String name) { TypeToSchema converter = new TypeToSchema(ImmutableMap.of(type, name)); TypeUtil.visit(type, converter); return ImmutableMap.copyOf(converter.getConversionMap()); }
public static <T> T visit(Schema schema, SchemaVisitor<T> visitor) { return visitor.schema(schema, visit(schema.asStruct(), visitor)); }
public static List<Record> generate(Schema schema, int numRecords, long seed) { RandomDataGenerator generator = new RandomDataGenerator(schema, seed); List<Record> records = Lists.newArrayListWithExpectedSize(numRecords); for (int i = 0; i < numRecords; i += 1) { records.add((Record) TypeUtil.visit(schema, generator)); } return records; }
public static List<Record> generateList(Schema schema, int numRecords, long seed) { RandomDataGenerator generator = new RandomDataGenerator(schema, seed); List<Record> records = Lists.newArrayListWithExpectedSize(numRecords); for (int i = 0; i < numRecords; i += 1) { records.add((Record) TypeUtil.visit(schema, generator)); } return records; }
public static List<Record> generate(Schema schema, int numRecords, long seed) { RandomDataGenerator generator = new RandomDataGenerator(seed); List<Record> records = Lists.newArrayListWithExpectedSize(numRecords); for (int i = 0; i < numRecords; i += 1) { records.add((Record) TypeUtil.visit(schema, generator)); } return records; }
private static Schema applyChanges(Schema schema, List<Integer> deletes, Map<Integer, Types.NestedField> updates, Multimap<Integer, Types.NestedField> adds) { Types.StructType struct = TypeUtil .visit(schema, new ApplyChanges(deletes, updates, adds)) .asNestedType().asStructType(); return new Schema(struct.fields()); }
static Schema fixup(Schema schema, Schema referenceSchema) { return new Schema(TypeUtil.visit(schema, new FixupTypes(referenceSchema)).asStructType().fields()); }
/** * Assigns fresh ids from the {@link NextID nextId function} for all fields in a schema. * * @param schema a schema * @param nextId an id assignment function * @return an structurally identical schema with new ids assigned by the nextId function */ public static Schema assignFreshIds(Schema schema, NextID nextId) { return new Schema(TypeUtil .visit(schema.asStruct(), new AssignFreshIds(nextId)) .asNestedType() .fields()); }