LOG.log(Level.FINEST, "Starting local job {0}", t.getIdentifier()); "/" + t.getIdentifier() + "-" + System.currentTimeMillis() + "/"); .addClientConfiguration(this.configurationSerializer.fromString(t.getConfiguration())) .setClientRemoteIdentifier(t.getRemoteId()) .setJobIdentifier(t.getIdentifier()).build(); .setErrorHandlerRID(t.getRemoteId()) .setLaunchID(t.getIdentifier()) .setConfigurationFileName(this.filenames.getDriverConfigurationPath()) .setClassPath(this.classpath.getDriverClasspath())
public Builder mergeFrom(com.microsoft.reef.proto.ClientRuntimeProtocol.JobSubmissionProto other) { if (other == com.microsoft.reef.proto.ClientRuntimeProtocol.JobSubmissionProto.getDefaultInstance()) return this; if (other.hasIdentifier()) { bitField0_ |= 0x00000001; identifier_ = other.identifier_; onChanged(); if (other.hasRemoteId()) { bitField0_ |= 0x00000002; remoteId_ = other.remoteId_; onChanged(); if (other.hasConfiguration()) { bitField0_ |= 0x00000004; configuration_ = other.configuration_; onChanged(); if (other.hasUserName()) { bitField0_ |= 0x00000008; userName_ = other.userName_; onChanged(); if (other.hasDriverMemory()) { setDriverMemory(other.getDriverMemory()); if (other.hasPriority()) { setPriority(other.getPriority()); if (other.hasQueue()) { bitField0_ |= 0x00000040;
hash = (19 * hash) + getDescriptorForType().hashCode(); if (hasIdentifier()) { hash = (37 * hash) + IDENTIFIER_FIELD_NUMBER; hash = (53 * hash) + getIdentifier().hashCode(); if (hasRemoteId()) { hash = (37 * hash) + REMOTE_ID_FIELD_NUMBER; hash = (53 * hash) + getRemoteId().hashCode(); if (hasConfiguration()) { hash = (37 * hash) + CONFIGURATION_FIELD_NUMBER; hash = (53 * hash) + getConfiguration().hashCode(); if (hasUserName()) { hash = (37 * hash) + USER_NAME_FIELD_NUMBER; hash = (53 * hash) + getUserName().hashCode(); if (hasDriverMemory()) { hash = (37 * hash) + DRIVER_MEMORY_FIELD_NUMBER; hash = (53 * hash) + getDriverMemory(); if (hasPriority()) { hash = (37 * hash) + PRIORITY_FIELD_NUMBER; hash = (53 * hash) + getPriority(); if (hasQueue()) { hash = (37 * hash) + QUEUE_FIELD_NUMBER; hash = (53 * hash) + getQueue().hashCode();
@Override public void onNext(final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto) { LOG.log(Level.FINEST, "Submitting job with ID [{0}]", jobSubmissionProto.getIdentifier()); "reef-job-" + jobSubmissionProto.getIdentifier()); .setErrorHandlerRID(jobSubmissionProto.getRemoteId()) .setLaunchID(jobSubmissionProto.getIdentifier()) .setConfigurationFileName(this.filenames.getDriverConfigurationPath()) .setClassPath(this.classpath.getDriverClasspath())
result = result && (hasIdentifier() == other.hasIdentifier()); if (hasIdentifier()) { result = result && getIdentifier() .equals(other.getIdentifier()); result = result && (hasRemoteId() == other.hasRemoteId()); if (hasRemoteId()) { result = result && getRemoteId() .equals(other.getRemoteId()); result = result && (hasConfiguration() == other.hasConfiguration()); if (hasConfiguration()) { result = result && getConfiguration() .equals(other.getConfiguration()); result = result && (hasUserName() == other.hasUserName()); if (hasUserName()) { result = result && getUserName() .equals(other.getUserName()); result = result && (hasDriverMemory() == other.hasDriverMemory()); if (hasDriverMemory()) { result = result && (getDriverMemory() == other.getDriverMemory()); result = result && (hasPriority() == other.hasPriority()); if (hasPriority()) { result = result && (getPriority() == other.getPriority());
.setApplicationName(jobSubmissionProto.getIdentifier()) .setResource(getResource(jobSubmissionProto)) .setContainerInfo(new ContainerInfo()
for (final ReefServiceProtos.FileResourceProto frp : jobSubmissionProto.getGlobalFileList()) { final File f = new File(frp.getPath()); if (frp.getType() == ReefServiceProtos.FileType.LIB) { for (final ReefServiceProtos.FileResourceProto frp : jobSubmissionProto.getLocalFileList()) { final File f = new File(frp.getPath()); if (frp.getType() == ReefServiceProtos.FileType.LIB) {
public File createJobSubmissionJAR( final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto, final Configuration driverConfiguration) throws IOException { // Copy all files to a local job submission folder final File jobSubmissionFolder = makejobSubmissionFolder(); LOG.log(Level.FINE, "Staging submission in {0}", jobSubmissionFolder); final File localFolder = new File(jobSubmissionFolder, this.fileNames.getLocalFolderName()); final File globalFolder = new File(jobSubmissionFolder, this.fileNames.getGlobalFolderName()); this.copy(jobSubmissionProto.getGlobalFileList(), globalFolder); this.copy(jobSubmissionProto.getLocalFileList(), localFolder); // Store the Driver Configuration in the JAR file. this.configurationSerializer.toFile( driverConfiguration, new File(localFolder, this.fileNames.getDriverConfigurationName())); // Create a JAR File for the submission final File jarFile = File.createTempFile(this.fileNames.getJobFolderPrefix(), this.fileNames.getJarFileSuffix()); LOG.log(Level.FINE, "Creating job submission jar file: {0}", jarFile); new JARFileMaker(jarFile).addChildren(jobSubmissionFolder).close(); if (this.deleteTempFilesOnExit) { LOG.log(Level.FINE, "Deleting the temporary job folder [{0}] and marking the jar file [{1}] for deletion after the JVM exits.", new Object[]{jobSubmissionFolder.getAbsolutePath(), jarFile.getAbsolutePath()}); jobSubmissionFolder.delete(); jarFile.deleteOnExit(); } else { LOG.log(Level.FINE, "Keeping the temporary job folder [{0}] and jar file [{1}] available after job submission.", new Object[]{jobSubmissionFolder.getAbsolutePath(), jarFile.getAbsolutePath()}); } return jarFile; }
/** * Extract the desired driver memory from jobSubmissionProto. * <p/> * returns maxMemory if that desired amount is more than maxMemory */ private int getMemory(final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto, final int maxMemory) { final int amMemory; final int requestedMemory = jobSubmissionProto.getDriverMemory(); if (requestedMemory <= maxMemory) { amMemory = requestedMemory; } else { LOG.log(Level.WARNING, "Requested {0}MB of memory for the driver. " + "The max on this YARN installation is {1}. " + "Using {1} as the memory for the driver.", new Object[]{requestedMemory, maxMemory}); amMemory = maxMemory; } return amMemory; }
/** * Assembles the Driver configuration. */ private Configuration makeDriverConfiguration( final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto, final Path jobFolderPath) throws IOException { Configuration config = this.configurationSerializer.fromString(jobSubmissionProto.getConfiguration()); final String userBoundJobSubmissionDirectory = config.getNamedParameter((NamedParameterNode<?>)config.getClassHierarchy().getNode(ReflectionUtilities.getFullName(DriverJobSubmissionDirectory.class))); LOG.log(Level.FINE, "user bound job submission Directory: " + userBoundJobSubmissionDirectory); final String finalJobFolderPath = (userBoundJobSubmissionDirectory == null || userBoundJobSubmissionDirectory.isEmpty()) ? jobFolderPath.toString() : userBoundJobSubmissionDirectory; return Configurations.merge( YarnDriverConfiguration.CONF .set(YarnDriverConfiguration.JOB_SUBMISSION_DIRECTORY, finalJobFolderPath) .set(YarnDriverConfiguration.JOB_IDENTIFIER, jobSubmissionProto.getIdentifier()) .set(YarnDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, jobSubmissionProto.getRemoteId()) .set(YarnDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack) .build(), this.configurationSerializer.fromString(jobSubmissionProto.getConfiguration())); }
/** * Assembles the command to execute the Driver in list form. */ private List<String> getCommandList( final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto) { return new JavaLaunchCommandBuilder() .setJavaPath("%JAVA_HOME%/bin/java") .setErrorHandlerRID(jobSubmissionProto.getRemoteId()) .setLaunchID(jobSubmissionProto.getIdentifier()) .setConfigurationFileName(this.filenames.getDriverConfigurationPath()) .setClassPath(this.classpath.getDriverClasspath()) .setMemory(jobSubmissionProto.getDriverMemory()) .setStandardErr(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStderrFileName()) .setStandardOut(ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + this.filenames.getDriverStdoutFileName()) .build(); }
/** * <code>required string identifier = 1;</code> * * <pre> * the job identifier * </pre> */ public Builder clearIdentifier() { bitField0_ = (bitField0_ & ~0x00000001); identifier_ = getDefaultInstance().getIdentifier(); onChanged(); return this; } /**
private Configuration makeDriverConfiguration( final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto, final String applicationId, final String jobFolderURL) throws IOException { final Configuration hdinsightDriverConfiguration = HDInsightDriverConfiguration.CONF .set(HDInsightDriverConfiguration.JOB_IDENTIFIER, applicationId) .set(HDInsightDriverConfiguration.JOB_SUBMISSION_DIRECTORY, jobFolderURL) .set(HDInsightDriverConfiguration.JVM_HEAP_SLACK, this.jvmHeapSlack) .build(); return Configurations.merge( this.configurationSerializer.fromString(jobSubmissionProto.getConfiguration()), hdinsightDriverConfiguration); } }
/** * <code>required string configuration = 5;</code> * * <pre> * the runtime configuration * </pre> */ public Builder clearConfiguration() { bitField0_ = (bitField0_ & ~0x00000004); configuration_ = getDefaultInstance().getConfiguration(); onChanged(); return this; } /**
/** * <code>required string remote_id = 2;</code> * * <pre> * the remote identifier * </pre> */ public Builder clearRemoteId() { bitField0_ = (bitField0_ & ~0x00000002); remoteId_ = getDefaultInstance().getRemoteId(); onChanged(); return this; } /**
/** * Extract the queue name from the jobSubmissionProto or return default if none is set. * <p/> * TODO: Revisit this. We also have a named parameter for the queue in YarnClientConfiguration. */ private final String getQueue(final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto, final String defaultQueue) { return jobSubmissionProto.hasQueue() && !jobSubmissionProto.getQueue().isEmpty() ? jobSubmissionProto.getQueue() : defaultQueue; }
private Priority getPriority(final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto) { return Priority.newInstance( jobSubmissionProto.hasPriority() ? jobSubmissionProto.getPriority() : 0); }
/** * Extracts the resource demands from the jobSubmissionProto. */ private final Resource getResource( final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto) { return new Resource() .setMemory(String.valueOf(jobSubmissionProto.getDriverMemory())) .setvCores("1"); }
/** * <code>optional string queue = 10;</code> */ public Builder clearQueue() { bitField0_ = (bitField0_ & ~0x00000040); queue_ = getDefaultInstance().getQueue(); onChanged(); return this; } /**