/** * Local configurations work in the same JVM and have no problems with improperly formatted files * on classpath (eg. directories with .class files or empty directories). Prepare files for * staging only when using remote cluster (passing the master address explicitly). */ private static void prepareFilesToStageForRemoteClusterExecution(SparkPipelineOptions options) { if (!options.getSparkMaster().matches("local\\[?\\d*\\]?")) { options.setFilesToStage( PipelineResources.prepareFilesForStaging( options.getFilesToStage(), options.getTempLocation())); } }
/** * Local configurations work in the same JVM and have no problems with improperly formatted files * on classpath (eg. directories with .class files or empty directories). Prepare files for * staging only when using remote cluster (passing the master address explicitly). */ private static void prepareFilesToStageForRemoteClusterExecution(FlinkPipelineOptions options) { if (!options.getFlinkMaster().matches("\\[auto\\]|\\[collection\\]|\\[local\\]")) { options.setFilesToStage( PipelineResources.prepareFilesForStaging( options.getFilesToStage(), options.getTempLocation())); } }
/** * Local configurations work in the same JVM and have no problems with improperly formatted files * on classpath (eg. directories with .class files or empty directories). Prepare files for * staging only when using remote cluster (passing the master address explicitly). */ private static void prepareFilesToStageForRemoteClusterExecution(FlinkPipelineOptions options) { if (!options.getFlinkMaster().matches("\\[auto\\]|\\[collection\\]|\\[local\\]")) { options.setFilesToStage( PipelineResources.prepareFilesForStaging( options.getFilesToStage(), MoreObjects.firstNonNull( options.getTempLocation(), System.getProperty("java.io.tmpdir")))); } }
@Test public void testPackagingDirectoryResourceToJarFile() throws IOException { String directoryPath = tmpFolder.newFolder().getAbsolutePath(); String temporaryLocation = tmpFolder.newFolder().getAbsolutePath(); List<String> filesToStage = new ArrayList<>(); filesToStage.add(directoryPath); List<String> result = PipelineResources.prepareFilesForStaging(filesToStage, temporaryLocation); assertTrue(new File(result.get(0)).exists()); assertTrue(result.get(0).matches(".*\\.jar")); }
@Test public void testRemovingNonexistentFilesFromFilesToStage() throws IOException { String nonexistentFilePath = tmpFolder.getRoot().getPath() + "/nonexistent/file"; String existingFilePath = tmpFolder.newFile("existingFile").getAbsolutePath(); String temporaryLocation = tmpFolder.newFolder().getAbsolutePath(); List<String> filesToStage = Arrays.asList(nonexistentFilePath, existingFilePath); List<String> expectedFilesToStage = Arrays.asList(existingFilePath); List<String> result = PipelineResources.prepareFilesForStaging(filesToStage, temporaryLocation); assertThat(result, is(expectedFilesToStage)); }
@Test public void testIfThrowsWhenThereIsNoTemporaryFolderForJars() throws IOException { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Please provide temporary location for storing the jar files."); List<String> filesToStage = new ArrayList<>(); filesToStage.add(tmpFolder.newFolder().getAbsolutePath()); PipelineResources.prepareFilesForStaging(filesToStage, null); } }