/** * @param config * @param param * @param values * @param <P> * @return * @deprecated in 0.2 this really should be in Tang. * See <a href="https://github.com/Microsoft-CISL/TANG/issues/164">Tang #164</a> for details. */ @Deprecated public static <P extends Param> ConfigurationModule addAll( ConfigurationModule config, final P param, final Iterable<String> values) { for (final String val : values) { config = config.set(param, val); } return config; }
/** * Adds the file names of this FileSet to the given field of the given ConfigurationModule. * * @param input the ConfigurationModule to fill out * @param field the field to add the files in this set to. * @return the filled out ConfigurationModule */ final ConfigurationModule addNamesTo(final ConfigurationModule input, final OptionalParameter<String> field) { ConfigurationModule result = input; for (final String fileName : this.fileNames()) { result = result.set(field, fileName); } return result; } }
/** * Instantiate a launcher for the given Configuration. * * @param runtimeConfiguration the resourcemanager configuration to be used * @return a DriverLauncher based on the given resourcemanager configuration * @throws BindException on configuration errors * @throws InjectionException on configuration errors */ public static DriverLauncher getLauncher( final Configuration runtimeConfiguration) throws BindException, InjectionException { final Configuration clientConfiguration = ClientConfiguration.CONF .set(ClientConfiguration.ON_JOB_RUNNING, RunningJobHandler.class) .set(ClientConfiguration.ON_JOB_COMPLETED, CompletedJobHandler.class) .set(ClientConfiguration.ON_JOB_FAILED, FailedJobHandler.class) .set(ClientConfiguration.ON_RUNTIME_ERROR, RuntimeErrorHandler.class) .build(); return Tang.Factory.getTang() .newInjector(runtimeConfiguration, clientConfiguration) .getInstance(DriverLauncher.class); }
private Configuration makeDriverConfiguration( final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto, final String applicationId, final String jobFolderURL) throws IOException { final Configuration hdinsightDriverConfiguration = HDInsightDriverConfiguration.CONF .set(HDInsightDriverConfiguration.JOB_IDENTIFIER, applicationId) .set(HDInsightDriverConfiguration.JOB_SUBMISSION_DIRECTORY, jobFolderURL) .set(HDInsightDriverConfiguration.JVM_HEAP_SLACK, this.jvmHeapSlack) .build(); return Configurations.merge( this.configurationSerializer.fromString(jobSubmissionProto.getConfiguration()), hdinsightDriverConfiguration); } }
@Override public void submitTask(final Configuration taskConfiguration) { final Configuration contextConfiguration = ContextConfiguration.CONF .set(ContextConfiguration.IDENTIFIER, "RootContext_" + this.getId()) .build(); this.submitContextAndTask(contextConfiguration, taskConfiguration); }
/** * Assembles the Driver configuration. */ private Configuration makeDriverConfiguration( final ClientRuntimeProtocol.JobSubmissionProto jobSubmissionProto, final Path jobFolderPath) throws IOException { Configuration config = this.configurationSerializer.fromString(jobSubmissionProto.getConfiguration()); final String userBoundJobSubmissionDirectory = config.getNamedParameter((NamedParameterNode<?>)config.getClassHierarchy().getNode(ReflectionUtilities.getFullName(DriverJobSubmissionDirectory.class))); LOG.log(Level.FINE, "user bound job submission Directory: " + userBoundJobSubmissionDirectory); final String finalJobFolderPath = (userBoundJobSubmissionDirectory == null || userBoundJobSubmissionDirectory.isEmpty()) ? jobFolderPath.toString() : userBoundJobSubmissionDirectory; return Configurations.merge( YarnDriverConfiguration.CONF .set(YarnDriverConfiguration.JOB_SUBMISSION_DIRECTORY, finalJobFolderPath) .set(YarnDriverConfiguration.JOB_IDENTIFIER, jobSubmissionProto.getIdentifier()) .set(YarnDriverConfiguration.CLIENT_REMOTE_IDENTIFIER, jobSubmissionProto.getRemoteId()) .set(YarnDriverConfiguration.JVM_HEAP_SLACK, this.jvmSlack) .build(), this.configurationSerializer.fromString(jobSubmissionProto.getConfiguration())); }
try { final Configuration idConfiguration = ContextConfiguration.CONF .set(ContextConfiguration.IDENTIFIER, dataLoadingService.getComputeContextIdPrefix() + evaluatorsForComputeRequest) .build();
if (renewFailedEvaluators) { driverConfiguration = this.driverConfigurationModule .set(DriverConfiguration.ON_DRIVER_STARTED, DataLoader.StartHandler.class) .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, DataLoader.EvaluatorAllocatedHandler.class) .set(DriverConfiguration.ON_EVALUATOR_FAILED, DataLoader.EvaluatorFailedHandler.class) .build(); } else { driverConfiguration = this.driverConfigurationModule .set(DriverConfiguration.ON_DRIVER_STARTED, DataLoader.StartHandler.class) .set(DriverConfiguration.ON_EVALUATOR_ALLOCATED, DataLoader.EvaluatorAllocatedHandler.class) .build();
@Override public Configuration getContextConfiguration(final AllocatedEvaluator allocatedEvaluator) { final NumberedSplit<InputSplit> numberedSplit = this.evaluatorToPartitionMapper.getInputSplit( allocatedEvaluator.getEvaluatorDescriptor().getNodeDescriptor().getName(), allocatedEvaluator.getId()); return ContextConfiguration.CONF .set(ContextConfiguration.IDENTIFIER, DATA_LOAD_CONTEXT_PREFIX + numberedSplit.getIndex()) .build(); }
try { final ConfigurationModule evaluatorConfigurationModule = EvaluatorConfiguration.CONF .set(EvaluatorConfiguration.APPLICATION_IDENTIFIER, this.jobIdentifier) .set(EvaluatorConfiguration.DRIVER_REMOTE_IDENTIFIER, this.remoteID) .set(EvaluatorConfiguration.EVALUATOR_IDENTIFIER, this.getId()); .set(EvaluatorConfiguration.ROOT_SERVICE_CONFIGURATION, encodedServiceConfigurationString) .set(EvaluatorConfiguration.ROOT_CONTEXT_CONFIGURATION, encodedContextConfigurationString); } else { .set(EvaluatorConfiguration.ROOT_CONTEXT_CONFIGURATION, encodedContextConfigurationString); final String encodedTaskConfigurationString = this.configurationSerializer.toString(taskConfiguration.get()); evaluatorConfiguration = contextConfigurationModule .set(EvaluatorConfiguration.TASK_CONFIGURATION, encodedTaskConfigurationString).build(); } else { evaluatorConfiguration = contextConfigurationModule.build();
@Override public Configuration getServiceConfiguration(final AllocatedEvaluator allocatedEvaluator) { try { final NumberedSplit<InputSplit> numberedSplit = this.evaluatorToPartitionMapper.getInputSplit( allocatedEvaluator.getEvaluatorDescriptor().getNodeDescriptor().getName(), allocatedEvaluator.getId()); final Configuration serviceConfiguration = ServiceConfiguration.CONF .set(ServiceConfiguration.SERVICES, this.inMemory ? InMemoryInputFormatDataSet.class : InputFormatDataSet.class) .build(); return Tang.Factory.getTang().newConfigurationBuilder(serviceConfiguration) .bindImplementation( DataSet.class, this.inMemory ? InMemoryInputFormatDataSet.class : InputFormatDataSet.class) .bindNamedParameter(JobConfExternalConstructor.InputFormatClass.class, inputFormatClass) .bindNamedParameter(JobConfExternalConstructor.InputPath.class, inputPath) .bindNamedParameter( InputSplitExternalConstructor.SerializedInputSplit.class, WritableSerializer.serialize(numberedSplit.getEntry())) .bindConstructor(InputSplit.class, InputSplitExternalConstructor.class) .bindConstructor(JobConf.class, JobConfExternalConstructor.class) .build(); } catch (final BindException ex) { final String evalId = allocatedEvaluator.getId(); final String msg = "Unable to create configuration for evaluator " + evalId; LOG.log(Level.WARNING, msg, ex); throw new RuntimeException(msg, ex); } }