Refine search
public boolean isApplicationAccepted(HiveConf conf, String applicationId) { if (applicationId == null) { return false; } YarnClient yarnClient = null; try { LOG.info("Trying to find " + applicationId); ApplicationId appId = getApplicationIDFromString(applicationId); yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); ApplicationReport appReport = yarnClient.getApplicationReport(appId); return appReport != null && appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED; } catch (Exception ex) { LOG.error("Failed getting application status for: " + applicationId + ": " + ex, ex); return false; } finally { if (yarnClient != null) { try { yarnClient.stop(); } catch (Exception ex) { LOG.error("Failed to stop yarn client: " + ex, ex); } } } } }
public static void killChildYarnJobs(Configuration conf, String tag) { try { if (tag == null) { return; } LOG.info("Killing yarn jobs using query tag:" + tag); Set<ApplicationId> childYarnJobs = getChildYarnJobs(conf, tag); if (!childYarnJobs.isEmpty()) { YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); for (ApplicationId app : childYarnJobs) { yarnClient.killApplication(app); } } } catch (IOException | YarnException ye) { LOG.warn("Exception occurred while killing child job({})", ye); } }
public GetNewApplicationResponse createAppMaster() throws YarnClientException { // Create application via yarnClient // Response is a new application ID along with cluster capacity info try { app = yarnClient.createApplication(); } catch (YarnException | IOException e) { throw new YarnClientException("Create application failed", e); } GetNewApplicationResponse response = app.getNewApplicationResponse(); appId = response.getApplicationId(); return response; }
public YarnRMClient(YarnConfiguration conf) { this.conf = conf; yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); }
JstormOnYarn(String appMasterMainClass, Configuration conf) { this.jstormClientContext.conf = conf; this.appMasterMainClass = appMasterMainClass; jstormClientContext.yarnClient = YarnClient.createYarnClient(); jstormClientContext.yarnClient.init(conf); jstormClientContext.opts = JstormYarnUtils.initClientOptions(); }
private void updateApplicationStatus() { if (yarnClient.isInState(Service.STATE.STARTED)) { final ApplicationReport applicationReport; try { applicationReport = yarnClient.getApplicationReport(yarnApplicationId); } catch (Exception e) { LOG.info("Could not retrieve the Yarn application report for {}.", yarnApplicationId); applicationStatus = ApplicationStatus.UNKNOWN; return; } YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState(); if (yarnApplicationState == YarnApplicationState.FAILED || yarnApplicationState == YarnApplicationState.KILLED) { applicationStatus = ApplicationStatus.FAILED; } else { applicationStatus = ApplicationStatus.SUCCEEDED; } } else { LOG.info("Yarn client is no longer in state STARTED. Stopping the Yarn application status monitor."); applicationStatusUpdateFuture.cancel(false); } } }
ApplicationReport report = jstormClientContext.yarnClient.getApplicationReport(appId); writename.createNewFile(); BufferedWriter out = new BufferedWriter(new FileWriter(writename)); out.write(report.getHost() + JOYConstants.NEW_LINE); out.write(report.getRpcPort() + JOYConstants.NEW_LINE); out.flush(); out.close(); + ", appId=" + appId.getId() + ", clientToAMToken=" + report.getClientToAMToken() + ", appDiagnostics=" + report.getDiagnostics() + ", appMasterHost=" + report.getHost()
jstormClientContext.yarnClient.start(); YarnClusterMetrics clusterMetrics = jstormClientContext.yarnClient.getYarnClusterMetrics(); LOG.info("Got Cluster metric info from ASM" + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); List<NodeReport> clusterNodeReports = jstormClientContext.yarnClient.getNodeReports( NodeState.RUNNING); LOG.info("Got Cluster node info from ASM, count is " + String.valueOf(clusterNodeReports.size())); QueueInfo queueInfo = jstormClientContext.yarnClient.getQueueInfo(this.jstormClientContext.amQueue); LOG.info("Queue info" + ", queueName=" + queueInfo.getQueueName() + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); List<QueueUserACLInfo> listAclInfo = jstormClientContext.yarnClient.getQueueAclsInfo(); for (QueueUserACLInfo aclInfo : listAclInfo) { for (QueueACL userAcl : aclInfo.getUserAcls()) { YarnClientApplication app = jstormClientContext.yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); int maxMem = appResponse.getMaximumResourceCapability().getMemory(); throw new IOException(JOYConstants.INSTANCE_DEPLOY_DIR_KEY + " is not set"); jstormClientContext.yarnClient.submitApplication(appContext);
YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); YarnClientApplication app = yarnClient.createApplication(); yarnClient.submitApplication(appContext); ApplicationReport appReport = yarnClient.getApplicationReport(appId); YarnApplicationState appState = appReport.getYarnApplicationState(); TimeUnit.SECONDS.sleep(1L); appReport = yarnClient.getApplicationReport(appId); && appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) log.log(Level.INFO, "Application {0} is ACCEPTED.", appId); appState = appReport.getYarnApplicationState();
YarnClient yc = YarnClient.createYarnClient(); yc.init(YARN_CONFIGURATION); yc.start(); List<ApplicationReport> apps = yc.getApplications(EnumSet.of(YarnApplicationState.RUNNING)); Assert.assertEquals(1, apps.size()); // Only one running ApplicationReport app = apps.get(0); Assert.assertEquals("MyCustomName", app.getName()); ApplicationId id = app.getApplicationId(); yc.killApplication(id); while (yc.getApplications(EnumSet.of(YarnApplicationState.KILLED)).size() == 0 && yc.getApplications(EnumSet.of(YarnApplicationState.FINISHED)).size() == 0) { sleep(500);
t.start(); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(new Configuration(conf)); yarnClient.start(); String hostName = NetUtils.getHostname(); String errorMessage = ""; while(!verified) { List<ApplicationReport> apps = yarnClient.getApplications(); if (apps.size() == 0 ) { Thread.sleep(10); if(appReport.getHost().equals("N/A")) { Thread.sleep(10); continue; + appReport.getHost() + "'. Expected rpc port to be '-1', was '" + appReport.getRpcPort() + "'."; if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) { verified = true;
File fp = File.createTempFile(appId.toString(), null); fp.deleteOnExit(); try (FileOutputStream output = new FileOutputStream(fp); appMasterEnv.put(YarnConfigKeys.ENV_TM_MEMORY, String.valueOf(clusterSpecification.getTaskManagerMemoryMB())); appMasterEnv.put(YarnConfigKeys.FLINK_JAR_PATH, remotePathJar.toString()); appMasterEnv.put(YarnConfigKeys.ENV_APP_ID, appId.toString()); appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_HOME_DIR, homeDir.toString()); appMasterEnv.put(YarnConfigKeys.ENV_CLIENT_SHIP_FILES, envShipFileList.toString()); Runtime.getRuntime().addShutdownHook(deploymentFailureHook); LOG.info("Submitting application master " + appId); yarnClient.submitApplication(appContext); loop: while (true) { try { report = yarnClient.getApplicationReport(appId); } catch (IOException e) { throw new YarnDeploymentException("Failed to deploy the cluster.", e); YarnApplicationState appState = report.getYarnApplicationState(); LOG.debug("Application State: {}", appState); switch(appState) {
void killExternalChildAppByTags(WorkflowAction action, YarnClient yarnClient, Configuration jobConf, String appExternalId) throws YarnException, IOException { for (ApplicationId id : LauncherMain.getChildYarnJobs(jobConf, ApplicationsRequestScope.ALL, action.getStartTime().getTime())) { if (finalAppStatusUndefined(yarnClient.getApplicationReport(id))) { try { LOG.info("Killing action {0}''s external child application {1} based on tags", action.getId(), id.toString()); yarnClient.killApplication(id); } catch (Exception e) { LOG.warn("Could not kill child of {0}, {1}", appExternalId, id, e); } } } }
YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication(); ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext(); appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE); this.yarnClient.submitApplication(appSubmissionContext); ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId); LOGGER.info("Application Name: " + applicationReport.getName()); LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl()); LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue());
@Override public void start(CallbackHandler resourceCallback, org.apache.hadoop.yarn.client.api.async.NMClientAsync.CallbackHandler nodeCallback ) { conf = new YarnConfiguration(); resourceMgr = AMRMClientAsync.createAMRMClientAsync(pollPeriodMs, resourceCallback); resourceMgr.init(conf); resourceMgr.start(); // Create the asynchronous node manager client nodeMgr = NMClientAsync.createNMClientAsync(nodeCallback); nodeMgr.init(conf); nodeMgr.start(); client = YarnClient.createYarnClient(); client.init(conf); client.start(); String appIdStr = System.getenv(DrillOnYarnConfig.APP_ID_ENV_VAR); if (appIdStr != null) { appId = ConverterUtils.toApplicationId(appIdStr); try { appReport = client.getApplicationReport(appId); } catch (YarnException | IOException e) { LOG.error( "Failed to get YARN applicaiton report for App ID: " + appIdStr, e); } } }
public static ApplicationReport getStartedAppInstanceByName(YarnClient clientRMService, String appName, String user, String excludeAppId) throws YarnException, IOException { List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED), EnumSet.of(YarnApplicationState.RUNNING, YarnApplicationState.ACCEPTED, YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING, YarnApplicationState.SUBMITTED)); // see whether there is an app with the app name and user name running for (ApplicationReport app : applications) { if (!app.getApplicationId().toString().equals(excludeAppId) && app.getName().equals(appName) && app.getUser().equals(user)) { return app; } } return null; }
private ApplicationReport assertRunningApp(ApplicationReport app) { ApplicationReport r; try { r = yarnClient.getApplicationReport(app.getApplicationId()); if (r.getYarnApplicationState() != YarnApplicationState.RUNNING) { String msg = String.format("Application %s not running (status %s)", r.getApplicationId().getId(), r.getYarnApplicationState()); throw new CliException(msg); } } catch (YarnException rmExc) { throw new CliException("Unable to determine application status", rmExc); } catch (IOException rmExc) { throw new CliException("Unable to determine application status", rmExc); } return r; }
yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); LOG.info("Requesting a new application from cluster with " + yarnClient.getYarnClusterMetrics().getNumNodeManagers() + " NodeManagers"); newAPP = yarnClient.createApplication();
assertTrue(yarnClient.isInState(Service.STATE.STARTED)); final YarnClient closableYarnClient = YarnClient.createYarnClient(); closableYarnClient.init(yarnConfiguration); closableYarnClient.start(); assertTrue(closableYarnClient.isInState(Service.STATE.STOPPED));
final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId); if (appReport.getFinalApplicationStatus() != FinalApplicationStatus.UNDEFINED) { applicationId, appReport.getFinalApplicationStatus()); throw new RuntimeException("The Yarn application " + applicationId + " doesn't run anymore."); final String host = appReport.getHost(); final int rpcPort = appReport.getRpcPort();