public static RegexMatcher matches(String regex) { return new RegexMatcher(regex); } }
@Test public void shouldPrepareFilesToStageWhenFlinkMasterIsSetExplicitly() throws IOException { FlinkPipelineOptions options = testPreparingResourcesToStage("localhost:8081"); assertThat(options.getFilesToStage().size(), is(1)); assertThat(options.getFilesToStage().get(0), matches(".*\\.jar")); }
@Test public void shouldPrepareFilesToStageWhenFlinkMasterIsSetExplicitly() throws IOException { FlinkPipelineOptions options = testPreparingResourcesToStage("localhost:8081"); assertThat(options.getFilesToStage().size(), is(1)); assertThat(options.getFilesToStage().get(0), matches(".*\\.jar")); }
@Test public void shouldNotPrepareFilesToStagewhenFlinkMasterIsSetToCollection() throws IOException { FlinkPipelineOptions options = testPreparingResourcesToStage("[collection]"); assertThat(options.getFilesToStage().size(), is(2)); assertThat(options.getFilesToStage(), everyItem(not(matches(".*\\.jar")))); }
@Test public void shouldNotPrepareFilesToStageWhenFlinkMasterIsSetToLocal() throws IOException { FlinkPipelineOptions options = testPreparingResourcesToStage("[local]"); assertThat(options.getFilesToStage().size(), is(2)); assertThat(options.getFilesToStage(), everyItem(not(matches(".*\\.jar")))); }
@Test public void shouldNotPrepareFilesToStageWhenFlinkMasterIsSetToLocal() throws IOException { FlinkPipelineOptions options = testPreparingResourcesToStage("[local]"); assertThat(options.getFilesToStage().size(), is(2)); assertThat(options.getFilesToStage(), everyItem(not(matches(".*\\.jar")))); }
@Test public void shouldNotPrepareFilesToStageWhenFlinkMasterIsSetToAuto() throws IOException { FlinkPipelineOptions options = testPreparingResourcesToStage("[auto]"); assertThat(options.getFilesToStage().size(), is(2)); assertThat(options.getFilesToStage(), everyItem(not(matches(".*\\.jar")))); }
@Test public void shouldNotPrepareFilesToStagewhenFlinkMasterIsSetToCollection() throws IOException { FlinkPipelineOptions options = testPreparingResourcesToStage("[collection]"); assertThat(options.getFilesToStage().size(), is(2)); assertThat(options.getFilesToStage(), everyItem(not(matches(".*\\.jar")))); }
@Test public void shouldNotPrepareFilesToStageWhenFlinkMasterIsSetToAuto() throws IOException { FlinkPipelineOptions options = testPreparingResourcesToStage("[auto]"); assertThat(options.getFilesToStage().size(), is(2)); assertThat(options.getFilesToStage(), everyItem(not(matches(".*\\.jar")))); }
@Test public void testPackageUploadWithExplicitPackageName() throws Exception { Pipe pipe = Pipe.open(); File tmpFile = makeFileWithContents("file.txt", "This is a test!"); final String overriddenName = "alias.txt"; when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))) .thenReturn( ImmutableList.of( StorageObjectOrIOException.create(new FileNotFoundException("some/path")))); when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink()); List<DataflowPackage> targets = defaultPackageUtil.stageClasspathElements( ImmutableList.of(overriddenName + "=" + tmpFile.getAbsolutePath()), STAGING_PATH, createOptions); DataflowPackage target = Iterables.getOnlyElement(targets); verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class)); verify(mockGcsUtil).create(any(GcsPath.class), anyString()); verifyNoMoreInteractions(mockGcsUtil); assertThat(target.getName(), equalTo(overriddenName)); assertThat( target.getLocation(), RegexMatcher.matches(STAGING_PATH + "file-" + HASH_PATTERN + ".txt")); }
@Test public void testPackageUploadWithEmptyDirectorySucceeds() throws Exception { Pipe pipe = Pipe.open(); File tmpDirectory = tmpFolder.newFolder("folder"); when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))) .thenReturn( ImmutableList.of( StorageObjectOrIOException.create(new FileNotFoundException("some/path")))); when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink()); List<DataflowPackage> targets = defaultPackageUtil.stageClasspathElements( ImmutableList.of(tmpDirectory.getAbsolutePath()), STAGING_PATH, createOptions); DataflowPackage target = Iterables.getOnlyElement(targets); verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class)); verify(mockGcsUtil).create(any(GcsPath.class), anyString()); verifyNoMoreInteractions(mockGcsUtil); assertThat(target.getName(), RegexMatcher.matches("folder-" + HASH_PATTERN + ".jar")); assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName())); try (ZipInputStream zipInputStream = new ZipInputStream(Channels.newInputStream(pipe.source()))) { assertNull(zipInputStream.getNextEntry()); } }
@Test public void testFileWithExtensionPackageNamingAndSize() throws Exception { String contents = "This is a test!"; File tmpFile = makeFileWithContents("file.txt", contents); PackageAttributes attr = makePackageAttributes(tmpFile, null); DataflowPackage target = attr.getDestination(); assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN + ".txt")); assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName())); assertThat(attr.getSize(), equalTo((long) contents.length())); }
@Test public void testPackageUploadWithFileSucceeds() throws Exception { Pipe pipe = Pipe.open(); String contents = "This is a test!"; File tmpFile = makeFileWithContents("file.txt", contents); when(mockGcsUtil.getObjects(anyListOf(GcsPath.class))) .thenReturn( ImmutableList.of( StorageObjectOrIOException.create(new FileNotFoundException("some/path")))); when(mockGcsUtil.create(any(GcsPath.class), anyString())).thenReturn(pipe.sink()); List<DataflowPackage> targets = defaultPackageUtil.stageClasspathElements( ImmutableList.of(tmpFile.getAbsolutePath()), STAGING_PATH, createOptions); DataflowPackage target = Iterables.getOnlyElement(targets); verify(mockGcsUtil).getObjects(anyListOf(GcsPath.class)); verify(mockGcsUtil).create(any(GcsPath.class), anyString()); verifyNoMoreInteractions(mockGcsUtil); assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN + ".txt")); assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName())); assertThat( new LineReader(Channels.newReader(pipe.source(), StandardCharsets.UTF_8.name())).readLine(), equalTo(contents)); }
@Test public void testPackageNamingWithDirectory() throws Exception { File tmpDirectory = tmpFolder.newFolder("folder"); DataflowPackage target = makePackageAttributes(tmpDirectory, null).getDestination(); assertThat(target.getName(), RegexMatcher.matches("folder-" + HASH_PATTERN + ".jar")); assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName())); }
@Test public void testPackageNamingWithFileNoExtension() throws Exception { File tmpFile = makeFileWithContents("file", "This is a test!"); DataflowPackage target = makePackageAttributes(tmpFile, null).getDestination(); assertThat(target.getName(), RegexMatcher.matches("file-" + HASH_PATTERN)); assertThat(target.getLocation(), equalTo(STAGING_PATH + target.getName())); }