public File createJobSubmissionJAR( final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto, final Configuration driverConfiguration) throws IOException { // Copy all files to a local job submission folder final File jobSubmissionFolder = makejobSubmissionFolder(); LOG.log(Level.FINE, "Staging submission in {0}", jobSubmissionFolder); final File localFolder = new File(jobSubmissionFolder, this.fileNames.getLocalFolderName()); final File globalFolder = new File(jobSubmissionFolder, this.fileNames.getGlobalFolderName()); this.copy(jobSubmissionProto.getGlobalFileList(), globalFolder); this.copy(jobSubmissionProto.getLocalFileList(), localFolder); // Store the Driver Configuration in the JAR file. this.configurationSerializer.toFile( driverConfiguration, new File(localFolder, this.fileNames.getDriverConfigurationName())); // Create a JAR File for the submission final File jarFile = File.createTempFile(this.fileNames.getJobFolderPrefix(), this.fileNames.getJarFileSuffix()); LOG.log(Level.FINE, "Creating job submission jar file: {0}", jarFile); new JARFileMaker(jarFile).addChildren(jobSubmissionFolder).close(); if (this.deleteTempFilesOnExit) { LOG.log(Level.FINE, "Deleting the temporary job folder [{0}] and marking the jar file [{1}] for deletion after the JVM exits.", new Object[]{jobSubmissionFolder.getAbsolutePath(), jarFile.getAbsolutePath()}); jobSubmissionFolder.delete(); jarFile.deleteOnExit(); } else { LOG.log(Level.FINE, "Keeping the temporary job folder [{0}] and jar file [{1}] available after job submission.", new Object[]{jobSubmissionFolder.getAbsolutePath(), jarFile.getAbsolutePath()}); } return jarFile; }
public static void copy(final Iterable<ReefServiceProtos.FileResourceProto> files, final File destinationFolder) { if (!destinationFolder.exists()) { destinationFolder.mkdirs(); } for (final ReefServiceProtos.FileResourceProto fileProto : files) { final File sourceFile = toFile(fileProto); final File destinationFile = new File(destinationFolder, fileProto.getName()); if (destinationFile.exists()) { LOG.log(Level.FINEST, "Will not add {0} to the job jar because another file with the same name was already added.", sourceFile.getAbsolutePath() ); } else { try { java.nio.file.Files.copy(sourceFile.toPath(), destinationFile.toPath()); } catch (final IOException e) { final String message = new StringBuilder("Copy of file [") .append(sourceFile.getAbsolutePath()) .append("] to [") .append(destinationFile.getAbsolutePath()) .append("] failed.") .toString(); throw new RuntimeException(message, e); } } } }
this.jobJarMaker.createJobSubmissionJAR(jobSubmissionProto, driverConfiguration);
JobJarMaker.copy(resourceLaunchProto.getFileList(), localStagingFolder);
this.jobJarMaker.createJobSubmissionJAR(jobSubmissionProto, driverConfiguration);