@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getJobFlowId() == null) ? 0 : getJobFlowId().hashCode()); hashCode = prime * hashCode + ((getSteps() == null) ? 0 : getSteps().hashCode()); return hashCode; }
/** * Constructs a new AddJobFlowStepsRequest object. Callers should use the setter or fluent setter (with...) methods * to initialize any additional object members. * * @param jobFlowId * A string that uniquely identifies the job flow. This identifier is returned by <a>RunJobFlow</a> and can * also be obtained from <a>ListClusters</a>. * @param steps * A list of <a>StepConfig</a> to be executed by the job flow. */ public AddJobFlowStepsRequest(String jobFlowId, java.util.List<StepConfig> steps) { setJobFlowId(jobFlowId); setSteps(steps); }
private Submitter existingClusterSubmitter(AmazonElasticMapReduce emr, String tag, StepCompiler stepCompiler, String clusterId, Filer filer) { return () -> { List<String> stepIds = pollingRetryExecutor(state, "submission") .retryUnless(AmazonServiceException.class, Aws::isDeterministicException) .withRetryInterval(DurationInterval.of(Duration.ofSeconds(30), Duration.ofMinutes(5))) .runOnce(new TypeReference<List<String>>() {}, s -> { RemoteFile runner = prepareRunner(filer, tag); // Compile steps stepCompiler.compile(runner); // Stage files to S3 filer.stageFiles(); AddJobFlowStepsRequest request = new AddJobFlowStepsRequest() .withJobFlowId(clusterId) .withSteps(stepCompiler.stepConfigs()); int steps = request.getSteps().size(); logger.info("Submitting {} EMR step(s) to {}", steps, clusterId); AddJobFlowStepsResult result = emr.addJobFlowSteps(request); logSubmittedSteps(clusterId, steps, i -> request.getSteps().get(i).getName(), i -> result.getStepIds().get(i)); return ImmutableList.copyOf(result.getStepIds()); }); return SubmissionResult.ofExistingCluster(clusterId, stepIds); }; }
@Override public void runJob(MavenManagedJar defaultJar, List<String> jarArgs) throws Exception { String jarLocation=defaultJar.s3JarLocation(awsSoftwareBucket); count++; String stepName = UUID.randomUUID().toString(); StepConfig step = new StepConfig(stepName,new HadoopJarStepConfig(jarLocation).withArgs(jarArgs)); emrClient.addJobFlowSteps(new AddJobFlowStepsRequest() .withJobFlowId(runningCluster) .withSteps(step)); Thread.sleep(5000); // Enough time to transition out of WAITING? pollClusterForCompletion(runningCluster, stepName); }
@Override public void addStepToExistingJobFlow( String stagingS3FileUrl, String stagingS3BucketUrl, String stepType, String mainClass, AbstractAmazonJobEntry jobEntry ) { this.alive = jobEntry.getAlive(); this.hadoopJobFlowId = jobEntry.getHadoopJobFlowId(); setStepsFromCluster(); List<StepConfig> steps = initSteps( stagingS3FileUrl, stepType, mainClass, jobEntry ); AddJobFlowStepsRequest addJobFlowStepsRequest = new AddJobFlowStepsRequest(); addJobFlowStepsRequest.setJobFlowId( hadoopJobFlowId ); addJobFlowStepsRequest.setSteps( steps ); emrClient.addJobFlowSteps( addJobFlowStepsRequest ); stepId = getSpecifiedRunningStep(); }
@Override public String addEmrStep(String clusterId, StepConfig emrStepConfig, AwsParamsDto awsParamsDto) throws Exception { List<StepConfig> steps = new ArrayList<>(); steps.add(emrStepConfig); // Add the job flow request AddJobFlowStepsRequest jobFlowStepRequest = new AddJobFlowStepsRequest(clusterId, steps); List<String> emrStepIds = emrOperations.addJobFlowStepsRequest(getEmrClient(awsParamsDto), jobFlowStepRequest); return emrStepIds.get(0); }
/** * Constructs a new AddJobFlowStepsRequest object. Callers should use the setter or fluent setter (with...) methods * to initialize any additional object members. * * @param jobFlowId * A string that uniquely identifies the job flow. This identifier is returned by <a>RunJobFlow</a> and can * also be obtained from <a>ListClusters</a>. */ public AddJobFlowStepsRequest(String jobFlowId) { setJobFlowId(jobFlowId); }
/** * <p> * A list of <a>StepConfig</a> to be executed by the job flow. * </p> * * @param steps * A list of <a>StepConfig</a> to be executed by the job flow. * @return Returns a reference to this object so that method calls can be chained together. */ public AddJobFlowStepsRequest withSteps(java.util.Collection<StepConfig> steps) { setSteps(steps); return this; }
@Override public String addEmrStep(String clusterId, StepConfig emrStepConfig, AwsParamsDto awsParamsDto) throws Exception { List<StepConfig> steps = new ArrayList<>(); steps.add(emrStepConfig); // Add the job flow request AddJobFlowStepsRequest jobFlowStepRequest = new AddJobFlowStepsRequest(clusterId, steps); List<String> emrStepIds = emrOperations.addJobFlowStepsRequest(getEmrClient(awsParamsDto), jobFlowStepRequest); return emrStepIds.get(0); }
/** * <p> * A string that uniquely identifies the job flow. This identifier is returned by <a>RunJobFlow</a> and can also be * obtained from <a>ListClusters</a>. * </p> * * @param jobFlowId * A string that uniquely identifies the job flow. This identifier is returned by <a>RunJobFlow</a> and can * also be obtained from <a>ListClusters</a>. * @return Returns a reference to this object so that method calls can be chained together. */ public AddJobFlowStepsRequest withJobFlowId(String jobFlowId) { setJobFlowId(jobFlowId); return this; }
/** * <p> * A list of <a>StepConfig</a> to be executed by the job flow. * </p> * <p> * <b>NOTE:</b> This method appends the values to the existing list (if any). Use * {@link #setSteps(java.util.Collection)} or {@link #withSteps(java.util.Collection)} if you want to override the * existing values. * </p> * * @param steps * A list of <a>StepConfig</a> to be executed by the job flow. * @return Returns a reference to this object so that method calls can be chained together. */ public AddJobFlowStepsRequest withSteps(StepConfig... steps) { if (this.steps == null) { setSteps(new com.amazonaws.internal.SdkInternalList<StepConfig>(steps.length)); } for (StepConfig ele : steps) { this.steps.add(ele); } return this; }
@Override public boolean equals(Object obj) { if (this == obj) return true; if (obj == null) return false; if (obj instanceof AddJobFlowStepsRequest == false) return false; AddJobFlowStepsRequest other = (AddJobFlowStepsRequest) obj; if (other.getJobFlowId() == null ^ this.getJobFlowId() == null) return false; if (other.getJobFlowId() != null && other.getJobFlowId().equals(this.getJobFlowId()) == false) return false; if (other.getSteps() == null ^ this.getSteps() == null) return false; if (other.getSteps() != null && other.getSteps().equals(this.getSteps()) == false) return false; return true; }
"-d", "windowTime=tablaparametro")); AddJobFlowStepsRequest jobFlow = new AddJobFlowStepsRequest().withJobFlowId(rb.getString("jobflowID")) .withSteps(stepHive);
/** * Constructs a new AddJobFlowStepsRequest object. Callers should use the setter or fluent setter (with...) methods * to initialize any additional object members. * * @param jobFlowId * A string that uniquely identifies the job flow. This identifier is returned by <a>RunJobFlow</a> and can * also be obtained from <a>ListClusters</a>. * @param steps * A list of <a>StepConfig</a> to be executed by the job flow. */ public AddJobFlowStepsRequest(String jobFlowId, java.util.List<StepConfig> steps) { setJobFlowId(jobFlowId); setSteps(steps); }
/** * Constructs a new AddJobFlowStepsRequest object. Callers should use the setter or fluent setter (with...) methods * to initialize any additional object members. * * @param jobFlowId * A string that uniquely identifies the job flow. This identifier is returned by <a>RunJobFlow</a> and can * also be obtained from <a>ListClusters</a>. */ public AddJobFlowStepsRequest(String jobFlowId) { setJobFlowId(jobFlowId); }
/** * <p> * A list of <a>StepConfig</a> to be executed by the job flow. * </p> * * @param steps * A list of <a>StepConfig</a> to be executed by the job flow. * @return Returns a reference to this object so that method calls can be chained together. */ public AddJobFlowStepsRequest withSteps(java.util.Collection<StepConfig> steps) { setSteps(steps); return this; }
/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getJobFlowId() != null) sb.append("JobFlowId: ").append(getJobFlowId()).append(","); if (getSteps() != null) sb.append("Steps: ").append(getSteps()); sb.append("}"); return sb.toString(); }
/** * <p> * A string that uniquely identifies the job flow. This identifier is returned by <a>RunJobFlow</a> and can also be * obtained from <a>ListClusters</a>. * </p> * * @param jobFlowId * A string that uniquely identifies the job flow. This identifier is returned by <a>RunJobFlow</a> and can * also be obtained from <a>ListClusters</a>. * @return Returns a reference to this object so that method calls can be chained together. */ public AddJobFlowStepsRequest withJobFlowId(String jobFlowId) { setJobFlowId(jobFlowId); return this; }
/** * <p> * A list of <a>StepConfig</a> to be executed by the job flow. * </p> * <p> * <b>NOTE:</b> This method appends the values to the existing list (if any). Use * {@link #setSteps(java.util.Collection)} or {@link #withSteps(java.util.Collection)} if you want to override the * existing values. * </p> * * @param steps * A list of <a>StepConfig</a> to be executed by the job flow. * @return Returns a reference to this object so that method calls can be chained together. */ public AddJobFlowStepsRequest withSteps(StepConfig... steps) { if (this.steps == null) { setSteps(new com.amazonaws.internal.SdkInternalList<StepConfig>(steps.length)); } for (StepConfig ele : steps) { this.steps.add(ele); } return this; }
/** * Marshall the given parameter object. */ public void marshall(AddJobFlowStepsRequest addJobFlowStepsRequest, ProtocolMarshaller protocolMarshaller) { if (addJobFlowStepsRequest == null) { throw new SdkClientException("Invalid argument passed to marshall(...)"); } try { protocolMarshaller.marshall(addJobFlowStepsRequest.getJobFlowId(), JOBFLOWID_BINDING); protocolMarshaller.marshall(addJobFlowStepsRequest.getSteps(), STEPS_BINDING); } catch (Exception e) { throw new SdkClientException("Unable to marshall request to JSON: " + e.getMessage(), e); } }