protected FileBasedSource<byte[]> getSource() { return CompressedSource.from(new TFRecordSource(getFilepattern())) .withCompression(getCompression()); }
@VisibleForTesting BoundedSource<T> createSource() { return CompressedSource.from( new XmlSource<>( StaticValueProvider.of(getFileOrPatternSpec()), getConfiguration(), 1L)) .withCompression(getCompression()); }
protected FileBasedSource<String> getSource() { return CompressedSource.from( new TextSource( getFilepattern(), getMatchConfiguration().getEmptyMatchTreatment(), getDelimiter())) .withCompression(getCompression()); }
@ProcessElement public void process(ProcessContext c) throws IOException { ReadableFile file = c.element().getKey(); OffsetRange range = c.element().getValue(); FileBasedSource<T> source = CompressedSource.from(createSource.apply(file.getMetadata().resourceId().toString())) .withCompression(file.getCompression()); try (BoundedSource.BoundedReader<T> reader = source .createForSubrangeOfFile(file.getMetadata(), range.getFrom(), range.getTo()) .createReader(c.getPipelineOptions())) { for (boolean more = reader.start(); more; more = reader.advance()) { c.output(reader.getCurrent()); } } } }