Refine search
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getName() == null) ? 0 : getName().hashCode()); hashCode = prime * hashCode + ((getLogUri() == null) ? 0 : getLogUri().hashCode()); hashCode = prime * hashCode + ((getAdditionalInfo() == null) ? 0 : getAdditionalInfo().hashCode()); hashCode = prime * hashCode + ((getAmiVersion() == null) ? 0 : getAmiVersion().hashCode()); hashCode = prime * hashCode + ((getReleaseLabel() == null) ? 0 : getReleaseLabel().hashCode()); hashCode = prime * hashCode + ((getInstances() == null) ? 0 : getInstances().hashCode()); hashCode = prime * hashCode + ((getSteps() == null) ? 0 : getSteps().hashCode()); hashCode = prime * hashCode + ((getBootstrapActions() == null) ? 0 : getBootstrapActions().hashCode()); hashCode = prime * hashCode + ((getSupportedProducts() == null) ? 0 : getSupportedProducts().hashCode()); hashCode = prime * hashCode + ((getNewSupportedProducts() == null) ? 0 : getNewSupportedProducts().hashCode()); hashCode = prime * hashCode + ((getApplications() == null) ? 0 : getApplications().hashCode()); hashCode = prime * hashCode + ((getConfigurations() == null) ? 0 : getConfigurations().hashCode()); hashCode = prime * hashCode + ((getVisibleToAllUsers() == null) ? 0 : getVisibleToAllUsers().hashCode()); hashCode = prime * hashCode + ((getJobFlowRole() == null) ? 0 : getJobFlowRole().hashCode()); hashCode = prime * hashCode + ((getServiceRole() == null) ? 0 : getServiceRole().hashCode()); hashCode = prime * hashCode + ((getTags() == null) ? 0 : getTags().hashCode()); hashCode = prime * hashCode + ((getSecurityConfiguration() == null) ? 0 : getSecurityConfiguration().hashCode()); hashCode = prime * hashCode + ((getAutoScalingRole() == null) ? 0 : getAutoScalingRole().hashCode()); hashCode = prime * hashCode + ((getScaleDownBehavior() == null) ? 0 : getScaleDownBehavior().hashCode()); hashCode = prime * hashCode + ((getCustomAmiId() == null) ? 0 : getCustomAmiId().hashCode()); hashCode = prime * hashCode + ((getEbsRootVolumeSize() == null) ? 0 : getEbsRootVolumeSize().hashCode()); hashCode = prime * hashCode + ((getRepoUpgradeOnBoot() == null) ? 0 : getRepoUpgradeOnBoot().hashCode()); hashCode = prime * hashCode + ((getKerberosAttributes() == null) ? 0 : getKerberosAttributes().hashCode()); return hashCode; }
/** * <p> * Applies to Amazon EMR releases 4.0 and later. A case-insensitive list of applications for Amazon EMR to install * and configure when launching the cluster. For a list of applications available for each Amazon EMR release * version, see the <a href="http://docs.aws.amazon.com/emr/latest/ReleaseGuide/">Amazon EMR Release Guide</a>. * </p> * <p> * <b>NOTE:</b> This method appends the values to the existing list (if any). Use * {@link #setApplications(java.util.Collection)} or {@link #withApplications(java.util.Collection)} if you want to * override the existing values. * </p> * * @param applications * Applies to Amazon EMR releases 4.0 and later. A case-insensitive list of applications for Amazon EMR to * install and configure when launching the cluster. For a list of applications available for each Amazon EMR * release version, see the <a href="http://docs.aws.amazon.com/emr/latest/ReleaseGuide/">Amazon EMR Release * Guide</a>. * @return Returns a reference to this object so that method calls can be chained together. */ public RunJobFlowRequest withApplications(Application... applications) { if (this.applications == null) { setApplications(new com.amazonaws.internal.SdkInternalList<Application>(applications.length)); } for (Application ele : applications) { this.applications.add(ele); } return this; }
/** * <p> * A list of bootstrap actions to run before Hadoop starts on the cluster nodes. * </p> * <p> * <b>NOTE:</b> This method appends the values to the existing list (if any). Use * {@link #setBootstrapActions(java.util.Collection)} or {@link #withBootstrapActions(java.util.Collection)} if you * want to override the existing values. * </p> * * @param bootstrapActions * A list of bootstrap actions to run before Hadoop starts on the cluster nodes. * @return Returns a reference to this object so that method calls can be chained together. */ public RunJobFlowRequest withBootstrapActions(BootstrapActionConfig... bootstrapActions) { if (this.bootstrapActions == null) { setBootstrapActions(new com.amazonaws.internal.SdkInternalList<BootstrapActionConfig>(bootstrapActions.length)); } for (BootstrapActionConfig ele : bootstrapActions) { this.bootstrapActions.add(ele); } return this; }
RunJobFlowRequest runJobFlowRequest = new RunJobFlowRequest(clusterName, getJobFlowInstancesConfig(emrClusterDefinition)); runJobFlowRequest.setReleaseLabel(emrClusterDefinition.getReleaseLabel()); runJobFlowRequest.setApplications(getApplications(emrClusterDefinitionApplications)); runJobFlowRequest.setConfigurations(getConfigurations(emrClusterDefinitionConfigurations)); runJobFlowRequest.setLogUri(emrClusterDefinition.getLogBucket()); runJobFlowRequest.setVisibleToAllUsers(emrClusterDefinition.isVisibleToAll()); runJobFlowRequest.setJobFlowRole(emrClusterDefinition.getEc2NodeIamProfileName()); runJobFlowRequest.setJobFlowRole(herdStringHelper.getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_EC2_NODE_IAM_PROFILE_NAME)); runJobFlowRequest.setServiceRole(emrClusterDefinition.getServiceIamRole()); runJobFlowRequest.setServiceRole(herdStringHelper.getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_SERVICE_IAM_ROLE_NAME)); runJobFlowRequest.setAmiVersion(emrClusterDefinition.getAmiVersion()); runJobFlowRequest.setAdditionalInfo(emrClusterDefinition.getAdditionalInfo()); runJobFlowRequest.setBootstrapActions(getBootstrapActionConfigList(emrClusterDefinition)); runJobFlowRequest.setSteps(getStepConfig(emrClusterDefinition)); runJobFlowRequest.setTags(getEmrTags(emrClusterDefinition));
RunJobFlowRequest request = new RunJobFlowRequest() .withName(cluster.get("name", String.class, "Digdag") + " (" + tag + ")") .withReleaseLabel(cluster.get("release", String.class, "emr-5.2.0")) .withSteps(stepConfigs) .withBootstrapActions(bootstrapActions) .withApplications(applicationConfigs) .withLogUri(cluster.get("logs", String.class, null)) .withJobFlowRole(cluster.get("cluster_role", String.class, "EMR_EC2_DefaultRole")) .withServiceRole(cluster.get("service_role", String.class, "EMR_DefaultRole")) .withTags(new Tag().withKey("DIGDAG_CLUSTER_ID").withValue(tag)) .withVisibleToAllUsers(cluster.get("visible", boolean.class, true)) .withConfigurations(configurations) .withInstances(new JobFlowInstancesConfig() .withInstanceGroups(ImmutableList.<InstanceGroupConfig>builder() .withKeepJobFlowAliveWhenNoSteps(!cluster.get("auto_terminate", boolean.class, true))); logger.info("Submitting EMR job with {} steps(s)", request.getSteps().size()); RunJobFlowResult result = emr.runJobFlow(request); logger.info("Submitted EMR job with {} step(s): {}", request.getSteps().size(), result.getJobFlowId(), result); return NewCluster.of(result.getJobFlowId(), request.getSteps().size());
) { RunJobFlowRequest runJobFlowRequest = new RunJobFlowRequest(); runJobFlowRequest.setName( jobEntry.getHadoopJobName() ); runJobFlowRequest.setReleaseLabel( jobEntry.getEmrRelease() ); runJobFlowRequest.setLogUri( stagingS3BucketUrl ); runJobFlowRequest.setInstances( instances ); runJobFlowRequest.setSteps( steps ); List<Application> applications = initApplications(); if ( applications.size() > 0 ) { runJobFlowRequest.setApplications( applications ); runJobFlowRequest.setBootstrapActions( stepBootstrapActions ); runJobFlowRequest.setJobFlowRole( EMR_EC2_DEFAULT_ROLE ); } else { runJobFlowRequest.setJobFlowRole( ec2Role ); runJobFlowRequest.setServiceRole( EMR_EFAULT_ROLE ); } else { runJobFlowRequest.setServiceRole( emrRole ); runJobFlowRequest.setVisibleToAllUsers( true );
RunJobFlowRequest request = new RunJobFlowRequest().withName("Spark Cluster").withSteps(steps).withServiceRole("EMR_DefaultRole").withJobFlowRole("EMR_EC2_DefaultRole") .withApplications(sparkApp).withReleaseLabel(emrVersion).withLogUri(getS3BucketLogsUrl()).withInstances(new JobFlowInstancesConfig().withEc2KeyName("spark") .withInstanceCount(instanceCount).withKeepJobFlowAliveWhenNoSteps(keepAlive).withMasterInstanceType(instanceType).withSlaveInstanceType(instanceType));
public String createPersistentCluster(String clusterName) throws Exception { StepConfig[] steps = { debugStep }; instances.setKeepJobFlowAliveWhenNoSteps(true); RunJobFlowRequest that=new RunJobFlowRequest() .withName(clusterName) .withSteps(steps) .withLogUri(awsLogUri) .withInstances(instances); RunJobFlowResult result = runJob(that); pollClusterForCompletion(result, Sets.union(doneStates, Sets.newHashSet("WAITING"))); return result.getJobFlowId(); }
JobFlowInstancesConfig jobFlowInstancesConfig = runJobFlowRequest.getInstances(); List<InstanceGroupConfig> instanceGroupConfigs = jobFlowInstancesConfig.getInstanceGroups(); assertEquals(2, instanceGroupConfigs.size()); runJobFlowRequest.getJobFlowRole()); assertEquals(herdStringHelper.getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_SERVICE_IAM_ROLE_NAME), runJobFlowRequest.getServiceRole()); List<StepConfig> stepConfigs = runJobFlowRequest.getSteps(); assertEquals(0, stepConfigs.size()); List<Tag> tags = runJobFlowRequest.getTags(); assertEquals(1, tags.size());
RunJobFlowRequest request = new RunJobFlowRequest() .withName("Hive Interactive") .withSteps(enabledebugging, installHive, runScript)
private MockEmrJobFlow createNewCluster(RunJobFlowRequest jobFlowRequest, String status, StatusChangeReason reason, StatusTimeline timeline) { MockEmrJobFlow cluster = new MockEmrJobFlow(); cluster.setJobFlowId(getNewJobFlowId()); cluster.setJobFlowName(jobFlowRequest.getName()); cluster.setStatus(status); cluster.setStatusTimeline(timeline); cluster.setStatusChangeReason(reason); emrClusters.put(cluster.getJobFlowId(), cluster); // Add the steps for (StepConfig stepConfig : jobFlowRequest.getSteps()) { addClusterStep(cluster.getJobFlowId(), stepConfig); } return cluster; }
/** * <p> * A list of bootstrap actions to run before Hadoop starts on the cluster nodes. * </p> * * @param bootstrapActions * A list of bootstrap actions to run before Hadoop starts on the cluster nodes. * @return Returns a reference to this object so that method calls can be chained together. */ public RunJobFlowRequest withBootstrapActions(java.util.Collection<BootstrapActionConfig> bootstrapActions) { setBootstrapActions(bootstrapActions); return this; }
/** * <p> * The location in Amazon S3 to write the log files of the job flow. If a value is not provided, logs are not * created. * </p> * * @param logUri * The location in Amazon S3 to write the log files of the job flow. If a value is not provided, logs are not * created. * @return Returns a reference to this object so that method calls can be chained together. */ public RunJobFlowRequest withLogUri(String logUri) { setLogUri(logUri); return this; }
/** * <p> * Also called instance profile and EC2 role. An IAM role for an EMR cluster. The EC2 instances of the cluster * assume this role. The default role is <code>EMR_EC2_DefaultRole</code>. In order to use the default role, you * must have already created it using the CLI or console. * </p> * * @param jobFlowRole * Also called instance profile and EC2 role. An IAM role for an EMR cluster. The EC2 instances of the * cluster assume this role. The default role is <code>EMR_EC2_DefaultRole</code>. In order to use the * default role, you must have already created it using the CLI or console. * @return Returns a reference to this object so that method calls can be chained together. */ public RunJobFlowRequest withJobFlowRole(String jobFlowRole) { setJobFlowRole(jobFlowRole); return this; }
/** * <p> * Applies to Amazon EMR releases 4.0 and later. A case-insensitive list of applications for Amazon EMR to install * and configure when launching the cluster. For a list of applications available for each Amazon EMR release * version, see the <a href="http://docs.aws.amazon.com/emr/latest/ReleaseGuide/">Amazon EMR Release Guide</a>. * </p> * * @param applications * Applies to Amazon EMR releases 4.0 and later. A case-insensitive list of applications for Amazon EMR to * install and configure when launching the cluster. For a list of applications available for each Amazon EMR * release version, see the <a href="http://docs.aws.amazon.com/emr/latest/ReleaseGuide/">Amazon EMR Release * Guide</a>. * @return Returns a reference to this object so that method calls can be chained together. */ public RunJobFlowRequest withApplications(java.util.Collection<Application> applications) { setApplications(applications); return this; }
/** * <p> * The Amazon EMR release label, which determines the version of open-source application packages installed on the * cluster. Release labels are in the form <code>emr-x.x.x</code>, where x.x.x is an Amazon EMR release version, for * example, <code>emr-5.14.0</code>. For more information about Amazon EMR release versions and included application * versions and features, see <a * href="http://docs.aws.amazon.com/emr/latest/ReleaseGuide/">http://docs.aws.amazon.com * /emr/latest/ReleaseGuide/</a>. The release label applies only to Amazon EMR releases versions 4.x and later. * Earlier versions use <code>AmiVersion</code>. * </p> * * @param releaseLabel * The Amazon EMR release label, which determines the version of open-source application packages installed * on the cluster. Release labels are in the form <code>emr-x.x.x</code>, where x.x.x is an Amazon EMR * release version, for example, <code>emr-5.14.0</code>. For more information about Amazon EMR release * versions and included application versions and features, see <a * href="http://docs.aws.amazon.com/emr/latest/ReleaseGuide/" * >http://docs.aws.amazon.com/emr/latest/ReleaseGuide/</a>. The release label applies only to Amazon EMR * releases versions 4.x and later. Earlier versions use <code>AmiVersion</code>. * @return Returns a reference to this object so that method calls can be chained together. */ public RunJobFlowRequest withReleaseLabel(String releaseLabel) { setReleaseLabel(releaseLabel); return this; }
@Override public String answer(InvocationOnMock invocation) throws Throwable { RunJobFlowRequest runJobFlowRequest = invocation.getArgument(1); // The oozie step should be skipped. assertEquals(0, runJobFlowRequest.getSteps().size()); return clusterId; } });
RunJobFlowRequest runJobFlowRequest = new RunJobFlowRequest(clusterName, getJobFlowInstancesConfig(emrClusterDefinition)); runJobFlowRequest.setReleaseLabel(emrClusterDefinition.getReleaseLabel()); runJobFlowRequest.setApplications(getApplications(emrClusterDefinitionApplications)); runJobFlowRequest.setConfigurations(getConfigurations(emrClusterDefinitionConfigurations)); runJobFlowRequest.setLogUri(emrClusterDefinition.getLogBucket()); runJobFlowRequest.setVisibleToAllUsers(emrClusterDefinition.isVisibleToAll()); runJobFlowRequest.setJobFlowRole(emrClusterDefinition.getEc2NodeIamProfileName()); runJobFlowRequest.setJobFlowRole(herdStringHelper.getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_EC2_NODE_IAM_PROFILE_NAME)); runJobFlowRequest.setServiceRole(emrClusterDefinition.getServiceIamRole()); runJobFlowRequest.setServiceRole(herdStringHelper.getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_SERVICE_IAM_ROLE_NAME)); runJobFlowRequest.setAmiVersion(emrClusterDefinition.getAmiVersion()); runJobFlowRequest.setAdditionalInfo(emrClusterDefinition.getAdditionalInfo()); runJobFlowRequest.setBootstrapActions(getBootstrapActionConfigList(emrClusterDefinition)); runJobFlowRequest.setSteps(getStepConfig(emrClusterDefinition)); runJobFlowRequest.setTags(getEmrTags(emrClusterDefinition));
RunJobFlowRequest request = new RunJobFlowRequest() .withName(name) .withApplications(new Application().withName("Spark")) .withConfigurations(new Configuration() .withClassification("yarn-site") .withProperties(Collections.singletonMap("yarn.nodemanager.aux-services", "mapreduce_shuffle,spark_shuffle"))) .withReleaseLabel("emr-5.0.0") .withServiceRole(emrConf.getServiceRole()) .withJobFlowRole(emrConf.getJobFlowRole()) .withInstances(new JobFlowInstancesConfig() .withEc2KeyName(name) .withAdditionalMasterSecurityGroups(emrConf.getAdditionalMasterSecurityGroup()) request.withLogUri(emrConf.getLogURI());
@Override public void runFlow(MavenManagedJar jar, Flow f,List<String> flowArgs) throws Exception { String jarLocation=jar.s3JarLocation(awsSoftwareBucket); List<StepConfig> steps = createEmrSteps(f, flowArgs, jarLocation); String jobName = computeJobName(flowArgs); RunJobFlowRequest that=new RunJobFlowRequest() .withName(jobName) .withBootstrapActions(bootstrapActions()) .withSteps(steps) .withLogUri(awsLogUri) .withInstances(instances); RunJobFlowResult result = runJob(that); String jobFlowId=result.getJobFlowId(); pollClusterForCompletion(result); fetchLogs.run(new String[] {jobFlowId}); alertService.alert("Cluster execution complete:\n"+jobName); }