/** * Convenience for {@link FileSourceBuilder#build(DistributedBiFunction)}. * Source emits lines to downstream without any transformation. */ public BatchSource<String> build() { return build((filename, line) -> line); }
/** * Returns a builder object that offers a step-by-step fluent API to build * a custom source to read files for the Pipeline API. The source reads * lines from files in a directory (but not its subdirectories). Using this * builder you can build {@linkplain FileSourceBuilder#build() batching} or * {@linkplain FileSourceBuilder#buildWatcher() streaming} reader. */ @Nonnull public static FileSourceBuilder filesBuilder(@Nonnull String directory) { return new FileSourceBuilder(directory); }
/** * Convenience for {@link FileSourceBuilder#buildWatcher(DistributedBiFunction)}. */ public StreamSource<String> buildWatcher() { return buildWatcher((filename, line) -> line); }
.sharedFileSystem(true) .glob("products.txt") .build((file, line) -> { Map.Entry<Integer, String> split = splitLine(line); return entry(split.getKey(), new Product(split.getKey(), split.getValue())); .sharedFileSystem(true) .glob("brokers.txt") .build((file, line) -> { Map.Entry<Integer, String> split = splitLine(line); return entry(split.getKey(), new Broker(split.getKey(), split.getValue()));
/** * A source to read all files in a directory in a batch way. * <p> * This method is a shortcut for: <pre>{@code * filesBuilder(directory) * .charset(UTF_8) * .glob(GLOB_WILDCARD) * .sharedFileSystem(false) * .mapToOutputFn((fileName, line) -> line) * .build() * }</pre> * <p> * If files are appended to while being read, the addition might or might * not be emitted or part of a line can be emitted. If files are modified * in more complex ways, the behavior is undefined. * * See {@link #filesBuilder(String)}. */ @Nonnull public static BatchSource<String> files(@Nonnull String directory) { return filesBuilder(directory).build(); }
/** * A source to stream lines added to files in a directory. This is a * streaming source, it will watch directory and emit lines as they are * appended to files in that directory. * <p> * This method is a shortcut for: <pre>{@code * filesBuilder(directory) * .charset(UTF_8) * .glob(GLOB_WILDCARD) * .sharedFileSystem(false) * .mapToOutputFn((fileName, line) -> line) * .buildWatcher() * }</pre> * * See {@link #filesBuilder(String)}. */ @Nonnull public static StreamSource<String> fileWatcher(@Nonnull String watchedDirectory) { return filesBuilder(watchedDirectory).buildWatcher(); }
.build(Util::entry));