/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getId() != null) sb.append("Id: ").append(getId()).append(","); if (getName() != null) sb.append("Name: ").append(getName()).append(","); if (getConfig() != null) sb.append("Config: ").append(getConfig()).append(","); if (getActionOnFailure() != null) sb.append("ActionOnFailure: ").append(getActionOnFailure()).append(","); if (getStatus() != null) sb.append("Status: ").append(getStatus()); sb.append("}"); return sb.toString(); }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getId() == null) ? 0 : getId().hashCode()); hashCode = prime * hashCode + ((getName() == null) ? 0 : getName().hashCode()); hashCode = prime * hashCode + ((getConfig() == null) ? 0 : getConfig().hashCode()); hashCode = prime * hashCode + ((getActionOnFailure() == null) ? 0 : getActionOnFailure().hashCode()); hashCode = prime * hashCode + ((getStatus() == null) ? 0 : getStatus().hashCode()); return hashCode; }
/** * Marshall the given parameter object. */ public void marshall(StepSummary stepSummary, ProtocolMarshaller protocolMarshaller) { if (stepSummary == null) { throw new SdkClientException("Invalid argument passed to marshall(...)"); } try { protocolMarshaller.marshall(stepSummary.getId(), ID_BINDING); protocolMarshaller.marshall(stepSummary.getName(), NAME_BINDING); protocolMarshaller.marshall(stepSummary.getConfig(), CONFIG_BINDING); protocolMarshaller.marshall(stepSummary.getActionOnFailure(), ACTIONONFAILURE_BINDING); protocolMarshaller.marshall(stepSummary.getStatus(), STATUS_BINDING); } catch (Exception e) { throw new SdkClientException("Unable to marshall request to JSON: " + e.getMessage(), e); } }
private String getSpecifiedRunningStep() { List<StepSummary> currentSteps = getSteps(); currentSteps.removeAll( stepSummaries ); if ( currentSteps.isEmpty() ) { return null; } return currentSteps.get( 0 ).getId(); }
private String getCurrentlyRunningStepId() { return getSteps().get( 0 ).getId(); }
/** * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be * redacted from this string using a placeholder value. * * @return A string representation of this object. * * @see java.lang.Object#toString() */ @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("{"); if (getId() != null) sb.append("Id: ").append(getId()).append(","); if (getName() != null) sb.append("Name: ").append(getName()).append(","); if (getConfig() != null) sb.append("Config: ").append(getConfig()).append(","); if (getActionOnFailure() != null) sb.append("ActionOnFailure: ").append(getActionOnFailure()).append(","); if (getStatus() != null) sb.append("Status: ").append(getStatus()); sb.append("}"); return sb.toString(); }
@Override public int hashCode() { final int prime = 31; int hashCode = 1; hashCode = prime * hashCode + ((getId() == null) ? 0 : getId().hashCode()); hashCode = prime * hashCode + ((getName() == null) ? 0 : getName().hashCode()); hashCode = prime * hashCode + ((getConfig() == null) ? 0 : getConfig().hashCode()); hashCode = prime * hashCode + ((getActionOnFailure() == null) ? 0 : getActionOnFailure().hashCode()); hashCode = prime * hashCode + ((getStatus() == null) ? 0 : getStatus().hashCode()); return hashCode; }
/** * Marshall the given parameter object. */ public void marshall(StepSummary stepSummary, ProtocolMarshaller protocolMarshaller) { if (stepSummary == null) { throw new SdkClientException("Invalid argument passed to marshall(...)"); } try { protocolMarshaller.marshall(stepSummary.getId(), ID_BINDING); protocolMarshaller.marshall(stepSummary.getName(), NAME_BINDING); protocolMarshaller.marshall(stepSummary.getConfig(), CONFIG_BINDING); protocolMarshaller.marshall(stepSummary.getActionOnFailure(), ACTIONONFAILURE_BINDING); protocolMarshaller.marshall(stepSummary.getStatus(), STATUS_BINDING); } catch (Exception e) { throw new SdkClientException("Unable to marshall request to JSON: " + e.getMessage(), e); } }
int stepIndex = submission.stepIds().indexOf(step.getId()); logger.info("Currently running EMR step {}/{}: {}: {}", stepIndex == -1 ? "?" : Integer.toString(stepIndex + 1), submission.stepIds().size(), step.getId(), step.getName()); }); FailureDetails details = status.getFailureDetails(); StepStateChangeReason reason = status.getStateChangeReason(); int stepIndex = submission.stepIds().indexOf(step.getId()); logger.error("EMR step {}/{}: {}: state: {}, reason: {}, details: {}", stepIndex == -1 ? "?" : Integer.toString(stepIndex + 1), submission.stepIds().size(), step.getId(), status.getState(), reason != null ? reason : "{}", details != null ? details : "{}");
private Submitter newClusterSubmitter(AmazonElasticMapReduce emr, String tag, StepCompiler stepCompiler, Config clusterConfig, Filer filer, ParameterCompiler parameterCompiler) { return () -> { // Start cluster NewCluster cluster = pollingRetryExecutor(state, "submission") .withRetryInterval(DurationInterval.of(Duration.ofSeconds(30), Duration.ofMinutes(5))) // TODO: EMR requests are not idempotent, thus retrying might produce duplicate cluster submissions. .retryUnless(AmazonServiceException.class, Aws::isDeterministicException) .runOnce(NewCluster.class, s -> submitNewClusterRequest(emr, tag, stepCompiler, clusterConfig, filer, parameterCompiler)); // Get submitted step IDs List<String> stepIds = pollingRetryExecutor(this.state, "steps") .withRetryInterval(DurationInterval.of(Duration.ofSeconds(30), Duration.ofMinutes(5))) .retryUnless(AmazonServiceException.class, Aws::isDeterministicException) .runOnce(new TypeReference<List<String>>() {}, s -> { List<StepSummary> steps = listSubmittedSteps(emr, tag, cluster); logSubmittedSteps(cluster.id(), cluster.steps(), i -> steps.get(i).getName(), i -> steps.get(i).getId()); return steps.stream().map(StepSummary::getId).collect(toList()); }); // Log cluster status while waiting for it to come up pollingWaiter(state, "bootstrap") .withWaitMessage("EMR cluster still booting") .withPollInterval(DurationInterval.of(Duration.ofSeconds(30), Duration.ofMinutes(5))) .awaitOnce(String.class, pollState -> checkClusterBootStatus(emr, cluster, pollState)); return SubmissionResult.ofNewCluster(cluster.id(), stepIds); }; }
/** * Builds EmrStep object from the EMR StepSummary. Fills in details if verbose=true. */ private EmrStep buildEmrStepFromAwsStepSummary(StepSummary stepSummary) { EmrStep emrStep = new EmrStep(); emrStep.setId(stepSummary.getId()); emrStep.setStepName(stepSummary.getName()); emrStep.setStatus(stepSummary.getStatus().getState()); return emrStep; }
/** * Builds EmrStep object from the EMR StepSummary. Fills in details if verbose=true. */ private EmrStep buildEmrStepFromAwsStepSummary(StepSummary stepSummary) { EmrStep emrStep = new EmrStep(); emrStep.setId(stepSummary.getId()); emrStep.setStepName(stepSummary.getName()); emrStep.setStatus(stepSummary.getStatus().getState()); return emrStep; }
/** * Builds EmrStep object from the EMR step. Fills in details if verbose=true. * * @param stepSummary The step summary * @param verbose The verbose flag * * @return EmrStep */ private EmrStep buildEmrStepFromAwsStep(StepSummary stepSummary, boolean verbose) { EmrStep emrStep = new EmrStep(); emrStep.setId(stepSummary.getId()); emrStep.setStepName(stepSummary.getName()); emrStep.setStatus(stepSummary.getStatus().getState()); if (verbose) { emrStep.setJarLocation(stepSummary.getConfig().getJar()); emrStep.setMainClass(stepSummary.getConfig().getMainClass()); emrStep.setScriptArguments(stepSummary.getConfig().getArgs()); emrStep.setContinueOnError(stepSummary.getActionOnFailure()); } return emrStep; }
/** * Builds EmrStep object from the EMR step. Fills in details if verbose=true. * * @param stepSummary The step summary * @param verbose The verbose flag * * @return EmrStep */ private EmrStep buildEmrStepFromAwsStep(StepSummary stepSummary, boolean verbose) { EmrStep emrStep = new EmrStep(); emrStep.setId(stepSummary.getId()); emrStep.setStepName(stepSummary.getName()); emrStep.setStatus(stepSummary.getStatus().getState()); if (verbose) { emrStep.setJarLocation(stepSummary.getConfig().getJar()); emrStep.setMainClass(stepSummary.getConfig().getMainClass()); emrStep.setScriptArguments(stepSummary.getConfig().getArgs()); emrStep.setContinueOnError(stepSummary.getActionOnFailure()); } return emrStep; }