/** * Writes to the provided number of shards. * * <p>Constraining the number of shards is likely to reduce the performance of a pipeline. * Setting this value is not recommended unless you require a specific number of output files. * * @param numShards the number of shards to use, or 0 to let the system decide. * @see ShardNameTemplate */ public Write withNumShards(int numShards) { checkArgument(numShards >= 0, "Number of shards %s must be >= 0", numShards); return toBuilder().setNumShards(numShards).build(); }
/** * A {@link PTransform} that writes a {@link PCollection} to TFRecord file (or multiple TFRecord * files matching a sharding pattern), with each element of the input collection encoded into its * own record. */ public static Write write() { return new AutoValue_TFRecordIO_Write.Builder() .setShardTemplate(null) .setFilenameSuffix(null) .setNumShards(0) .setCompression(Compression.UNCOMPRESSED) .build(); }
/** * Writes to output files using the specified compression type. * * <p>If no compression type is specified, the default is {@link Compression#UNCOMPRESSED}. See * {@link TFRecordIO.Read#withCompression} for more details. */ public Write withCompression(Compression compression) { return toBuilder().setCompression(compression).build(); }
/** * Writes to the file(s) with the given filename suffix. * * @see ShardNameTemplate */ public Write withSuffix(String suffix) { return toBuilder().setFilenameSuffix(suffix).build(); }
/** * Uses the given shard name template. * * @see ShardNameTemplate */ public Write withShardNameTemplate(String shardTemplate) { return toBuilder().setShardTemplate(shardTemplate).build(); }
/** Like {@link #to(ResourceId)}. */ @Experimental(Kind.FILESYSTEM) public Write toResource(ValueProvider<ResourceId> outputResource) { return toBuilder().setOutputPrefix(outputResource).build(); }