/** * A {@link PTransform} that writes a {@link PCollection} to an avro file (or multiple avro files * matching a sharding pattern), with each element of the input collection encoded into its own * record of type OutputT. * * <p>This version allows you to apply {@link AvroIO} writes to a PCollection of a custom type * {@link UserT}. A format mechanism that converts the input type {@link UserT} to the output type * that will be written to the file must be specified. If using a custom {@link * DynamicAvroDestinations} object this is done using {@link * DynamicAvroDestinations#formatRecord}, otherwise the {@link * AvroIO.TypedWrite#withFormatFunction} can be used to specify a format function. * * <p>The advantage of using a custom type is that is it allows a user-provided {@link * DynamicAvroDestinations} object, set via {@link AvroIO.Write#to(DynamicAvroDestinations)} to * examine the custom type when choosing a destination. * * <p>If the output type is {@link GenericRecord} use {@link #writeCustomTypeToGenericRecords()} * instead. */ public static <UserT, OutputT> TypedWrite<UserT, Void, OutputT> writeCustomType() { return AvroIO.<UserT, OutputT>defaultWriteBuilder().setGenericRecords(false).build(); }
/** Writes Avro records of the specified schema. */ public static Write<GenericRecord> writeGenericRecords(Schema schema) { return new Write<>( AvroIO.<GenericRecord, GenericRecord>defaultWriteBuilder() .setGenericRecords(true) .setSchema(schema) .build()); }
/** * Writes a {@link PCollection} to an Avro file (or multiple Avro files matching a sharding * pattern). */ public static <T> Write<T> write(Class<T> recordClass) { return new Write<>( AvroIO.<T, T>defaultWriteBuilder() .setGenericRecords(false) .setSchema(ReflectData.get().getSchema(recordClass)) .build()); }
/** * Similar to {@link #writeCustomType()}, but specialized for the case where the output type is * {@link GenericRecord}. A schema must be specified either in {@link * DynamicAvroDestinations#getSchema} or if not using dynamic destinations, by using {@link * TypedWrite#withSchema(Schema)}. */ public static <UserT> TypedWrite<UserT, Void, GenericRecord> writeCustomTypeToGenericRecords() { return AvroIO.<UserT, GenericRecord>defaultWriteBuilder().setGenericRecords(true).build(); }