public YarnApplicationState getState() { return report.getYarnApplicationState(); }
private static boolean isApplicationRunning(ApplicationReport app) { final YarnApplicationState yarnApplicationState = app.getYarnApplicationState(); return yarnApplicationState != YarnApplicationState.FINISHED && app.getYarnApplicationState() != YarnApplicationState.KILLED && app.getYarnApplicationState() != YarnApplicationState.FAILED; }
private void logApplicationReport(ApplicationReport appReport) { LOG.info("Application " + appReport.getApplicationId() + " finished with state " + appReport .getYarnApplicationState() + " and final state " + appReport .getFinalApplicationStatus() + " at " + appReport.getFinishTime()); if (appReport.getYarnApplicationState() == YarnApplicationState.FAILED) { LOG.warn("Application failed. Diagnostics " + appReport.getDiagnostics()); LOG.warn("If log aggregation is activated in the Hadoop cluster, we recommend to retrieve " + "the full application log using this command:" + System.lineSeparator() + "\tyarn logs -applicationId " + appReport.getApplicationId() + System.lineSeparator() + "(It sometimes takes a few seconds until the logs are aggregated)"); } }
private void updateApplicationStatus() { if (yarnClient.isInState(Service.STATE.STARTED)) { final ApplicationReport applicationReport; try { applicationReport = yarnClient.getApplicationReport(yarnApplicationId); } catch (Exception e) { LOG.info("Could not retrieve the Yarn application report for {}.", yarnApplicationId); applicationStatus = ApplicationStatus.UNKNOWN; return; } YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState(); if (yarnApplicationState == YarnApplicationState.FAILED || yarnApplicationState == YarnApplicationState.KILLED) { applicationStatus = ApplicationStatus.FAILED; } else { applicationStatus = ApplicationStatus.SUCCEEDED; } } else { LOG.info("Yarn client is no longer in state STARTED. Stopping the Yarn application status monitor."); applicationStatusUpdateFuture.cancel(false); } } }
/** * Wait for the application to enter one of the completion states. This is an * informal implementation useful for testing. * * @throws YarnClientException */ public void waitForCompletion() throws YarnClientException { ApplicationReport appReport; YarnApplicationState appState; for (;;) { appReport = getAppReport(); appState = appReport.getYarnApplicationState(); if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.KILLED || appState == YarnApplicationState.FAILED) { break; } try { Thread.sleep(100); } catch (InterruptedException e) { // Should never occur. } } System.out.println("Application " + appId + " finished with" + " state " + appState + " at " + appReport.getFinishTime()); }
public void showFinalStatus() { YarnApplicationState state = report.getYarnApplicationState(); if (state == YarnApplicationState.FAILED || state == YarnApplicationState.FINISHED) { FinalApplicationStatus status = report.getFinalApplicationStatus(); System.out.println("Final status: " + status.toString()); if (status != FinalApplicationStatus.SUCCEEDED) { String diag = report.getDiagnostics(); if (!DoYUtil.isBlank(diag)) { System.out.println("Diagnostics: " + diag); } } } }
@Subscribe public void handleApplicationReportArrivalEvent(ApplicationReportArrivalEvent applicationReportArrivalEvent) { ApplicationReport applicationReport = applicationReportArrivalEvent.getApplicationReport(); YarnApplicationState appState = applicationReport.getYarnApplicationState(); LOGGER.info("Gobblin Yarn application state: " + appState.toString()); // Reset the count on failures to get the ApplicationReport when there's one success this.getApplicationReportFailureCount.set(0); if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.FAILED || appState == YarnApplicationState.KILLED) { applicationCompleted = true; LOGGER.info("Gobblin Yarn application finished with final status: " + applicationReport.getFinalApplicationStatus().toString()); if (applicationReport.getFinalApplicationStatus() == FinalApplicationStatus.FAILED) { LOGGER.error("Gobblin Yarn application failed for the following reason: " + applicationReport.getDiagnostics()); } try { GobblinYarnAppLauncher.this.stop(); } catch (IOException ioe) { LOGGER.error("Failed to close the " + GobblinYarnAppLauncher.class.getSimpleName(), ioe); } catch (TimeoutException te) { LOGGER.error("Timeout in stopping the service manager", te); } finally { if (this.emailNotificationOnShutdown) { sendEmailOnShutdown(Optional.of(applicationReport)); } } } }
for (;;) { appReport = getAppReport(); appState = appReport.getYarnApplicationState(); attemptId = appReport.getCurrentApplicationAttemptId(); if (appState != YarnApplicationState.NEW
public void display(boolean verbose, boolean isNew) { YarnApplicationState state = report.getYarnApplicationState(); if (verbose || !isNew) { System.out.println("Application State: " + state.toString()); System.out.println("Host: " + report.getHost()); } if (verbose || !isNew) { System.out.println("Queue: " + report.getQueue()); System.out.println("User: " + report.getUser()); long startTime = report.getStartTime(); System.out.println("Start Time: " + DoYUtil.toIsoTime(startTime)); System.out.println("Application Name: " + report.getName()); } System.out.println("Tracking URL: " + report.getTrackingUrl()); if (isNew) { System.out.println("Application Master URL: " + getAmUrl()); } showFinalStatus(); }
public boolean isApplicationAccepted(HiveConf conf, String applicationId) { if (applicationId == null) { return false; } YarnClient yarnClient = null; try { LOG.info("Trying to find " + applicationId); ApplicationId appId = getApplicationIDFromString(applicationId); yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); ApplicationReport appReport = yarnClient.getApplicationReport(appId); return appReport != null && appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED; } catch (Exception ex) { LOG.error("Failed getting application status for: " + applicationId + ": " + ex, ex); return false; } finally { if (yarnClient != null) { try { yarnClient.stop(); } catch (Exception ex) { LOG.error("Failed to stop yarn client: " + ex, ex); } } } } }
@Test(enabled=false, groups = { "disabledOnTravis" }, dependsOnMethods = "testSetupAndSubmitApplication") public void testGetReconnectableApplicationId() throws Exception { Assert.assertEquals(this.gobblinYarnAppLauncher.getReconnectableApplicationId().get(), this.applicationId); this.yarnClient.killApplication(this.applicationId); Assert.assertEquals(yarnClient.getApplicationReport(applicationId).getYarnApplicationState(), YarnApplicationState.KILLED, "Application not killed"); // takes some time for kill to take effect and app master to go down Thread.sleep(5000); }
public static boolean isApplicationAccepted(HiveConf conf, String applicationId) { if (applicationId == null) { return false; } YarnClient yarnClient = null; try { ApplicationId appId = getApplicationIDFromString(applicationId); yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); ApplicationReport appReport = yarnClient.getApplicationReport(appId); return appReport != null && appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED; } catch (Exception ex) { LOG.error("Failed getting application status for: " + applicationId + ": " + ex, ex); return false; } finally { if (yarnClient != null) { try { yarnClient.stop(); } catch (Exception ex) { LOG.error("Failed to stop yarn client: " + ex, ex); } } } }
throw new YarnDeploymentException("Failed to deploy the cluster.", e); YarnApplicationState appState = report.getYarnApplicationState(); LOG.debug("Application State: {}", appState); switch(appState) {
+ ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) {
/** * For some yet unknown reason, hostname resolution for the ResourceManager in {@link MiniYARNCluster} * has some issue that causes the {@link YarnClient} not be able to connect and submit the Gobblin Yarn * application successfully. This works fine on local machine though. So disabling this and the test * below that depends on it on Travis-CI. */ @Test(enabled=false, groups = { "disabledOnTravis" }, dependsOnMethods = "testCreateHelixCluster") public void testSetupAndSubmitApplication() throws Exception { this.gobblinYarnAppLauncher.startYarnClient(); this.applicationId = this.gobblinYarnAppLauncher.setupAndSubmitApplication(); int i; // wait for application to come up for (i = 0; i < 120; i++) { if (yarnClient.getApplicationReport(applicationId).getYarnApplicationState() == YarnApplicationState.RUNNING) { break; } Thread.sleep(1000); } Assert.assertTrue(i < 120, "timed out waiting for RUNNING state"); // wait another 10 seconds and check state again to make sure that the application stays up Thread.sleep(10000); Assert.assertEquals(yarnClient.getApplicationReport(applicationId).getYarnApplicationState(), YarnApplicationState.RUNNING, "Application may have aborted"); }
/** * Sleep a bit between the tests (we are re-using the YARN cluster for the tests). */ @After public void sleep() throws IOException, YarnException { Deadline deadline = Deadline.now().plus(Duration.ofSeconds(10)); boolean isAnyJobRunning = yarnClient.getApplications().stream() .anyMatch(YarnTestBase::isApplicationRunning); while (deadline.hasTimeLeft() && isAnyJobRunning) { try { Thread.sleep(500); } catch (InterruptedException e) { Assert.fail("Should not happen"); } isAnyJobRunning = yarnClient.getApplications().stream() .anyMatch(YarnTestBase::isApplicationRunning); } if (isAnyJobRunning) { final List<String> runningApps = yarnClient.getApplications().stream() .filter(YarnTestBase::isApplicationRunning) .map(app -> "App " + app.getApplicationId() + " is in state " + app.getYarnApplicationState() + '.') .collect(Collectors.toList()); if (!runningApps.isEmpty()) { Assert.fail("There is at least one application on the cluster that is not finished." + runningApps); } } }
if (xlearningClient == null && isRunning.get()) { LOG.info("Application report for " + applicationId + " (state: " + applicationReport.getYarnApplicationState().toString() + ")"); xlearningClient = getAppMessageHandler(conf, applicationReport.getHost(), applicationReport.getRpcPort()); YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState(); FinalApplicationStatus finalApplicationStatus = applicationReport.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == yarnApplicationState) {
YarnApplicationState appState = appReport.getYarnApplicationState(); && appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) log.log(Level.INFO, "Application {0} is ACCEPTED.", appId); appState = appReport.getYarnApplicationState();
new AmInfo().setAppName(appReport.getName()).setAppType(appReport.getApplicationType())); appStatusBuilder.setAppStartTime(appReport.getStartTime()); switch (appReport.getYarnApplicationState()) { case NEW: case NEW_SAVING: default: throw new LlapStatusCliException(ExitCode.INTERNAL_ERROR, "Unknown Yarn Application State: " + appReport.getYarnApplicationState());
public static YarnApplicationState pollFinishedApplicationState(YarnClient client, ApplicationId appId) throws IOException, YarnException, InterruptedException { EnumSet<YarnApplicationState> finishedState = EnumSet.of(FINISHED, KILLED, FAILED); while (true) { ApplicationReport report = client.getApplicationReport(appId); YarnApplicationState state = report.getYarnApplicationState(); if (finishedState.contains(state)) { return state; } else { Thread.sleep(250); } } }