How to use

FileChecksumMatcher

in

org.apache.beam.sdk.testing

/**
 * Computes a checksum of the sharded file specified in the constructor. Not safe to call until
 * the writing is complete.
 */
private String getActualChecksum() {
 if (actualChecksum == null) {
  // Load output data
  List<String> outputs;
  try {
   outputs = shardedFile.readFilesWithRetries(Sleeper.DEFAULT, BACK_OFF_FACTORY.backoff());
  } catch (Exception e) {
   throw new RuntimeException(String.format("Failed to read from: %s", shardedFile), e);
  }
  // Verify outputs. Checksum is computed using SHA-1 algorithm
  actualChecksum = computeHash(outputs);
  LOG.debug("Generated checksum: {}", actualChecksum);
 }
 return actualChecksum;
}

@Override
public boolean matchesSafely(PipelineResult pipelineResult) {
 return getActualChecksum().equals(expectedChecksum);
}

@Test
public void testMatcherThatVerifiesSingleFile() throws IOException {
 File tmpFile = tmpFolder.newFile("result-000-of-001");
 Files.write("Test for file checksum verifier.", tmpFile, StandardCharsets.UTF_8);
 FileChecksumMatcher matcher =
   new FileChecksumMatcher("a8772322f5d7b851777f820fc79d050f9d302915", tmpFile.getPath());
 assertThat(pResult, matcher);
}

@Test
public void testMatcherThatVerifiesFileWithEmptyContent() throws IOException {
 File emptyFile = tmpFolder.newFile("result-000-of-001");
 Files.write("", emptyFile, StandardCharsets.UTF_8);
 FileChecksumMatcher matcher =
   new FileChecksumMatcher(
     "da39a3ee5e6b4b0d3255bfef95601890afd80709",
     tmpFolder.getRoot().toPath().resolve("*").toString());
 assertThat(pResult, matcher);
}

@Test
public void testPreconditionFilePathIsEmpty() {
 thrown.expect(IllegalArgumentException.class);
 thrown.expectMessage(containsString("Expected valid file path, but received"));
 new FileChecksumMatcher("checksumString", "");
}

 @Override
 public void describeMismatchSafely(PipelineResult pResult, Description description) {
  description.appendText("was (").appendText(getActualChecksum()).appendText(")");
 }
}

@Test
public void testPreconditionChecksumIsEmpty() throws IOException {
 String tmpPath = tmpFolder.newFile().getPath();
 thrown.expect(IllegalArgumentException.class);
 thrown.expectMessage(containsString("Expected valid checksum, but received"));
 new FileChecksumMatcher("", tmpPath);
}

@Test
public void testPreconditionChecksumIsNull() throws IOException {
 String tmpPath = tmpFolder.newFile().getPath();
 thrown.expect(IllegalArgumentException.class);
 thrown.expectMessage(containsString("Expected valid checksum, but received"));
 new FileChecksumMatcher(null, tmpPath);
}

@Test
public void testPreconditionShardTemplateIsNull() throws IOException {
 String tmpPath = tmpFolder.newFile().getPath();
 thrown.expect(NullPointerException.class);
 thrown.expectMessage(
   containsString(
     "Expected non-null shard pattern. "
       + "Please call the other constructor to use default pattern:"));
 new FileChecksumMatcher("checksumString", tmpPath, null);
}

 @Test
 public void testMatcherThatUsesCustomizedTemplate() throws Exception {
  // Customized template: resultSSS-totalNNN
  File tmpFile1 = tmpFolder.newFile("result0-total2");
  File tmpFile2 = tmpFolder.newFile("result1-total2");
  Files.write("To be or not to be, ", tmpFile1, StandardCharsets.UTF_8);
  Files.write("it is not a question.", tmpFile2, StandardCharsets.UTF_8);

  Pattern customizedTemplate =
    Pattern.compile("(?x) result (?<shardnum>\\d+) - total (?<numshards>\\d+)");
  FileChecksumMatcher matcher =
    new FileChecksumMatcher(
      "90552392c28396935fe4f123bd0b5c2d0f6260c8",
      tmpFolder.getRoot().toPath().resolve("*").toString(),
      customizedTemplate);

  assertThat(pResult, matcher);
 }
}

@Test
public void testMatcherThatVerifiesMultipleFiles() throws IOException {
 File tmpFile1 = tmpFolder.newFile("result-000-of-002");
 File tmpFile2 = tmpFolder.newFile("result-001-of-002");
 File tmpFile3 = tmpFolder.newFile("tmp");
 Files.write("To be or not to be, ", tmpFile1, StandardCharsets.UTF_8);
 Files.write("it is not a question.", tmpFile2, StandardCharsets.UTF_8);
 Files.write("tmp", tmpFile3, StandardCharsets.UTF_8);
 FileChecksumMatcher matcher =
   new FileChecksumMatcher(
     "90552392c28396935fe4f123bd0b5c2d0f6260c8",
     tmpFolder.getRoot().toPath().resolve("result-*").toString());
 assertThat(pResult, matcher);
}

 @Test
 public void testE2EWordCount() throws Exception {
  WordCountITOptions options = TestPipeline.testingPipelineOptions().as(WordCountITOptions.class);

  options.setInputFile(DEFAULT_INPUT);
  options.setOutput(
    FileSystems.matchNewResource(options.getTempRoot(), true)
      .resolve(
        String.format("WordCountIT-%tF-%<tH-%<tM-%<tS-%<tL", new Date()),
        StandardResolveOptions.RESOLVE_DIRECTORY)
      .resolve("output", StandardResolveOptions.RESOLVE_DIRECTORY)
      .resolve("results", StandardResolveOptions.RESOLVE_FILE)
      .toString());
  options.setOnSuccessMatcher(
    new FileChecksumMatcher(DEFAULT_OUTPUT_CHECKSUM, options.getOutput() + "*-of-*"));

  WordCount.runWordCount(options);
 }
}

 @Test
 public void testE2ETopWikiPages() throws Exception {
  TopWikipediaSessionsITOptions options =
    TestPipeline.testingPipelineOptions().as(TopWikipediaSessionsITOptions.class);

  options.setWikiInput(DEFAULT_INPUT_10_FILES);
  options.setOutput(
    FileSystems.matchNewResource(options.getTempRoot(), true)
      .resolve(
        String.format("topwikisessions-it-%tF-%<tH-%<tM-%<tS-%<tL", new Date()),
        StandardResolveOptions.RESOLVE_DIRECTORY)
      .resolve("output", StandardResolveOptions.RESOLVE_DIRECTORY)
      .resolve("results", StandardResolveOptions.RESOLVE_FILE)
      .toString());
  options.setOnSuccessMatcher(
    new FileChecksumMatcher(DEFAULT_OUTPUT_CHECKSUM, options.getOutput() + "*-of-*"));

  TopWikipediaSessions.run(options);
 }
}

 @Test
 public void testE2ETfIdf() throws Exception {
  TfIdfITOptions options = TestPipeline.testingPipelineOptions().as(TfIdfITOptions.class);
  options.setInput(DEFAULT_INPUT);
  options.setOutput(
    FileSystems.matchNewResource(options.getTempRoot(), true)
      .resolve(
        String.format("TfIdfIT-%tF-%<tH-%<tM-%<tS-%<tL", new Date()),
        StandardResolveOptions.RESOLVE_DIRECTORY)
      .resolve("output", StandardResolveOptions.RESOLVE_DIRECTORY)
      .resolve("results", StandardResolveOptions.RESOLVE_FILE)
      .toString());
  options.setOnSuccessMatcher(
    new FileChecksumMatcher(
      EXPECTED_OUTPUT_CHECKSUM, options.getOutput() + "*-of-*.csv", DEFAULT_SHARD_TEMPLATE));

  TfIdf.runTfIdf(options);
 }
}

new FileChecksumMatcher(
  VALUE_CHECKSUM, new FilePatternMatchingShardedFile(singleOutputPrefix + "*")),
new FileChecksumMatcher(
  VALUE_CHECKSUM, new FilePatternMatchingShardedFile(multiOutputPrefix + "*"))));