Refine search
ApplicationReport report = jstormClientContext.yarnClient.getApplicationReport(appId); writename.createNewFile(); BufferedWriter out = new BufferedWriter(new FileWriter(writename)); out.write(report.getHost() + JOYConstants.NEW_LINE); out.write(report.getRpcPort() + JOYConstants.NEW_LINE); out.flush(); out.close(); + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost() + ", appQueue=" + report.getQueue() + ", appMasterRpcPort=" + report.getRpcPort() + ", appStartTime=" + report.getStartTime() + ", yarnAppState=" + report.getYarnApplicationState().toString() + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() + ", appTrackingUrl=" + report.getTrackingUrl() + ", appUser=" + report.getUser()); YarnApplicationState state = report.getYarnApplicationState(); FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == state) { if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
private void logApplicationReport(ApplicationReport appReport) { LOG.info("Application " + appReport.getApplicationId() + " finished with state " + appReport .getYarnApplicationState() + " and final state " + appReport .getFinalApplicationStatus() + " at " + appReport.getFinishTime()); if (appReport.getYarnApplicationState() == YarnApplicationState.FAILED) { LOG.warn("Application failed. Diagnostics " + appReport.getDiagnostics()); LOG.warn("If log aggregation is activated in the Hadoop cluster, we recommend to retrieve " + "the full application log using this command:" + System.lineSeparator() + "\tyarn logs -applicationId " + appReport.getApplicationId() + System.lineSeparator() + "(It sometimes takes a few seconds until the logs are aggregated)"); } }
@VisibleForTesting Optional<ApplicationId> getReconnectableApplicationId() throws YarnException, IOException { List<ApplicationReport> applicationReports = this.yarnClient.getApplications(APPLICATION_TYPES, RECONNECTABLE_APPLICATION_STATES); if (applicationReports == null || applicationReports.isEmpty()) { return Optional.absent(); } // Try to find an application with a matching application name for (ApplicationReport applicationReport : applicationReports) { if (this.applicationName.equals(applicationReport.getName())) { return Optional.of(applicationReport.getApplicationId()); } } return Optional.absent(); }
public void showFinalStatus() { YarnApplicationState state = report.getYarnApplicationState(); if (state == YarnApplicationState.FAILED || state == YarnApplicationState.FINISHED) { FinalApplicationStatus status = report.getFinalApplicationStatus(); System.out.println("Final status: " + status.toString()); if (status != FinalApplicationStatus.SUCCEEDED) { String diag = report.getDiagnostics(); if (!DoYUtil.isBlank(diag)) { System.out.println("Diagnostics: " + diag); } } } }
public SerializedApplicationReport(ApplicationReport report) { this.applicationId = report.getApplicationId().toString(); ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId(); this.applicationAttemptId = attemptId != null ? attemptId.toString() : "N/A"; this.name = report.getName(); this.applicationType = report.getApplicationType(); this.user = report.getUser(); this.queue = report.getQueue(); this.host = report.getHost(); this.rpcPort = report.getRpcPort(); this.state = report.getYarnApplicationState().toString(); this.diagnostics = report.getDiagnostics(); this.startTime = report.getStartTime(); this.finishTime = report.getFinishTime(); FinalApplicationStatus appStatus = report.getFinalApplicationStatus(); this.finalStatus = appStatus == null ? "" : appStatus.toString(); this.progress = report.getProgress(); this.url = report.getTrackingUrl(); this.origTrackingUrl= report.getOriginalTrackingUrl(); }
public Collection<CloudAppInstanceInfo> getSubmittedApplications(String yarnApplicationId) { List<CloudAppInstanceInfo> appIds = new ArrayList<CloudAppInstanceInfo>(); for (ApplicationReport report : yarnClient.listApplications("DATAFLOW")) { if (report.getApplicationId().toString().equals(yarnApplicationId)) { appIds.add(new CloudAppInstanceInfo(report.getApplicationId().toString(), report.getName(), report.getYarnApplicationState().toString(), report.getOriginalTrackingUrl())); } } return appIds; }
public boolean isApplicationAccepted(HiveConf conf, String applicationId) { if (applicationId == null) { return false; } YarnClient yarnClient = null; try { LOG.info("Trying to find " + applicationId); ApplicationId appId = getApplicationIDFromString(applicationId); yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); ApplicationReport appReport = yarnClient.getApplicationReport(appId); return appReport != null && appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED; } catch (Exception ex) { LOG.error("Failed getting application status for: " + applicationId + ": " + ex, ex); return false; } finally { if (yarnClient != null) { try { yarnClient.stop(); } catch (Exception ex) { LOG.error("Failed to stop yarn client: " + ex, ex); } } } } }
public void display(boolean verbose, boolean isNew) { YarnApplicationState state = report.getYarnApplicationState(); if (verbose || !isNew) { System.out.println("Application State: " + state.toString()); System.out.println("Host: " + report.getHost()); } if (verbose || !isNew) { System.out.println("Queue: " + report.getQueue()); System.out.println("User: " + report.getUser()); long startTime = report.getStartTime(); System.out.println("Start Time: " + DoYUtil.toIsoTime(startTime)); System.out.println("Application Name: " + report.getName()); } System.out.println("Tracking URL: " + report.getTrackingUrl()); if (isNew) { System.out.println("Application Master URL: " + getAmUrl()); } showFinalStatus(); }
applicationId = appReport.getApplicationId().toString(); new AmInfo().setAppName(appReport.getName()).setAppType(appReport.getApplicationType())); appStatusBuilder.setAppStartTime(appReport.getStartTime()); switch (appReport.getYarnApplicationState()) { case NEW: case NEW_SAVING: case KILLED: appStatusBuilder.maybeCreateAndGetAmInfo().setAppId(applicationId); appStatusBuilder.setAppFinishTime(appReport.getFinishTime()); appStatusBuilder.setState(State.COMPLETE); default: throw new LlapStatusCliException(ExitCode.INTERNAL_ERROR, "Unknown Yarn Application State: " + appReport.getYarnApplicationState());
YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); ApplicationReport appReport = yarnClient.getApplicationReport(appId); YarnApplicationState appState = appReport.getYarnApplicationState(); && appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) log.log(Level.INFO, "Application {0} is ACCEPTED.", appId); appState = appReport.getYarnApplicationState();
static InstanceInfo extractInstanceInfo(String clusterName, ApplicationReport report) { InstanceMetadata md = getMetadata(report.getApplicationTags()); if (md == null) { return null; } ApplicationResourceUsageReport usage = report.getApplicationResourceUsageReport(); InstanceStatus stat = new InstanceStatus() .allocatedVCores((long) usage.getUsedResources().getVirtualCores()) .allocatedMB((long) usage.getUsedResources().getMemory()) .clusterId(clusterName) .applicationId(report.getApplicationId().toString()) .startedTime(report.getStartTime()) .runningContainers((long) usage.getNumUsedContainers()) .trackingUrl(report.getTrackingUrl()) .state(InstanceStatus.StateEnum.fromValue(report.getYarnApplicationState().toString())); return new InstanceInfo(clusterName, report.getApplicationId(), md, stat); } }
YarnClient yc = YarnClient.createYarnClient(); yc.init(YARN_CONFIGURATION); yc.start(); List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING)); Assert.assertEquals(1, apps.size()); // Only one running ApplicationReport app = apps.get(0); Assert.assertEquals("MyCustomName", app.getName()); ApplicationId id = app.getApplicationId(); yc.killApplication(id);
t.start(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(new Configuration(conf)); yarnClient.start(); String hostName = NetUtils.getHostname(); if(appReport.getHost().equals("N/A")) { Thread.sleep(10); continue; + appReport.getHost() + "'. Expected rpc port to be '-1', was '" + appReport.getRpcPort() + "'."; if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) { verified = true; if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) { break;
File fp = File.createTempFile(appId.toString(), null); fp.deleteOnExit(); try (FileOutputStream output = new FileOutputStream(fp); appMasterEnv.put(YarnConfigKeys.ENV_TM_MEMORY, String.valueOf(clusterSpecification.getTaskManagerMemoryMB())); appMasterEnv.put(YarnConfigKeys.FLINK_JAR_PATH, remotePathJar.toString()); appMasterEnv.put(YarnConfigKeys.ENV_APP_ID, appId.toString()); appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, homeDir.toString()); appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, envShipFileList.toString()); Runtime.getRuntime().addShutdownHook(deploymentFailureHook); LOG.info("Submitting application master " + appId); yarnClient.submitApplication(appContext); YarnApplicationState appState = report.getYarnApplicationState(); LOG.debug("Application State: {}", appState); switch(appState) { throw new YarnDeploymentException("The YARN application unexpectedly switched to state " + appState + " during deployment. \n" + "Diagnostics from YARN: " + report.getDiagnostics() + "\n" + "If log aggregation is enabled on your cluster, use this command to further investigate the issue:\n" + "yarn logs -applicationId " + appId);
if (applicationReport.isPresent()) { messageBuilder.append("\n"); messageBuilder.append("\tApplication ID: ").append(applicationReport.get().getApplicationId()).append("\n"); messageBuilder.append("\tApplication attempt ID: ") .append(applicationReport.get().getCurrentApplicationAttemptId()).append("\n"); messageBuilder.append("\tFinal application status: ").append(applicationReport.get().getFinalApplicationStatus()) .append("\n"); messageBuilder.append("\tStart time: ").append(applicationReport.get().getStartTime()).append("\n"); messageBuilder.append("\tFinish time: ").append(applicationReport.get().getFinishTime()).append("\n"); if (!Strings.isNullOrEmpty(applicationReport.get().getDiagnostics())) { messageBuilder.append("\tDiagnostics: ").append(applicationReport.get().getDiagnostics()).append("\n"); ApplicationResourceUsageReport resourceUsageReport = applicationReport.get().getApplicationResourceUsageReport(); if (resourceUsageReport != null) { messageBuilder.append("\tUsed containers: ").append(resourceUsageReport.getNumUsedContainers()).append("\n");
private boolean waitCompleted() throws IOException, YarnException { ApplicationReport applicationReport = getApplicationReport(applicationId, yarnClient); LOG.info("The url to track the job: " + applicationReport.getTrackingUrl()); while (true) { assert (applicationReport != null); if (xlearningClient == null && isRunning.get()) { LOG.info("Application report for " + applicationId + " (state: " + applicationReport.getYarnApplicationState().toString() + ")"); xlearningClient = getAppMessageHandler(conf, applicationReport.getHost(), applicationReport.getRpcPort()); YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState(); FinalApplicationStatus finalApplicationStatus = applicationReport.getFinalApplicationStatus(); if (YarnApplicationState.FINISHED == yarnApplicationState) { xlearningClient = null; return true; } else { LOG.info("Application has completed failed with YarnApplicationState=" + yarnApplicationState.toString() + " and FinalApplicationStatus=" + finalApplicationStatus.toString()); return false; xlearningClient = null; isRunning.set(false); LOG.info("Application has completed with YarnApplicationState=" + yarnApplicationState.toString() + " and FinalApplicationStatus=" + finalApplicationStatus.toString()); return false;
/** * convert an AM report to a string for diagnostics * @param report the report * @return the string value */ public static String reportToString(ApplicationReport report) { if (report == null) { return "Null application report"; } return "App " + report.getName() + "/" + report.getApplicationType() + "# " + report.getApplicationId() + " user " + report.getUser() + " is in state " + report.getYarnApplicationState() + " RPC: " + report.getHost() + ":" + report.getRpcPort() + " URL: " + report.getOriginalTrackingUrl(); }
@Override public String apply(final ApplicationReport input) { Preconditions.checkNotNull(input, "YARN application should be filled"); return input.getApplicationId().toString(); } });
YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication(); ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext(); appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE); this.yarnClient.submitApplication(appSubmissionContext); ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId); LOGGER.info("Application Name: " + applicationReport.getName()); LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl()); LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue());
private void setupMocks(final String mrJobId, final String wfExternalId, final String yarnApplicationId) throws HadoopAccessorException, IOException, URISyntaxException, InterruptedException, YarnException { when(hadoopJobIdFinder.find()).thenReturn(mrJobId); when(applicationReport.getApplicationType()).thenReturn("MAPREDUCE"); when(workflowActionBean.getWfId()).thenReturn("workflowId"); when(workflowActionBean.getExternalId()).thenReturn(wfExternalId); when(applicationReport.getYarnApplicationState()).thenReturn(YarnApplicationState.RUNNING); when(applicationId.toString()).thenReturn(yarnApplicationId); when(applicationReport.getApplicationId()).thenReturn(applicationId); when(reader.read()).thenReturn(Lists.newArrayList(applicationReport)); }