@Override public PCollection<String> expand(PCollection<String> input) { return input .apply(FileIO.matchAll().withConfiguration(getMatchConfiguration())) .apply( FileIO.readMatches() .withCompression(getCompression()) .withDirectoryTreatment(DirectoryTreatment.PROHIBIT)) .apply(readFiles().withDelimiter(getDelimiter())); }
@Test @Category(NeedsRunner.class) public void testReadFiles() throws IOException { Path tempFolderPath = tempFolder.getRoot().toPath(); writeToFile(TINY, tempFolder, "readAllTiny1.zip", ZIP); writeToFile(TINY, tempFolder, "readAllTiny2.txt", UNCOMPRESSED); writeToFile(LARGE, tempFolder, "readAllLarge1.zip", ZIP); writeToFile(LARGE, tempFolder, "readAllLarge2.txt", UNCOMPRESSED); PCollection<String> lines = p.apply( Create.of( tempFolderPath.resolve("readAllTiny*").toString(), tempFolderPath.resolve("readAllLarge*").toString())) .apply(FileIO.matchAll()) .apply(FileIO.readMatches().withCompression(AUTO)) .apply(TextIO.readFiles().withDesiredBundleSizeBytes(10)); PAssert.that(lines).containsInAnyOrder(Iterables.concat(TINY, TINY, LARGE, LARGE)); p.run(); }