public String getTokenIdentifier() { return fakeAppId.toString(); }
@VisibleForTesting void cleanUpAppWorkDirectory(ApplicationId applicationId) throws IOException { Path appWorkDir = GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, applicationId.toString()); if (this.fs.exists(appWorkDir)) { LOGGER.info("Deleting application working directory " + appWorkDir); this.fs.delete(appWorkDir, true); } }
private void writeYarnPropertiesFile( ApplicationId yarnApplicationId, int parallelism, @Nullable String dynamicProperties) { // file that we write into the conf/ dir containing the jobManager address and the dop. final File yarnPropertiesFile = getYarnPropertiesLocation(yarnPropertiesFileLocation); Properties yarnProps = new Properties(); yarnProps.setProperty(YARN_APPLICATION_ID_KEY, yarnApplicationId.toString()); if (parallelism > 0) { yarnProps.setProperty(YARN_PROPERTIES_PARALLELISM, Integer.toString(parallelism)); } // add dynamic properties if (dynamicProperties != null) { yarnProps.setProperty(YARN_PROPERTIES_DYNAMIC_PROPERTIES_STRING, dynamicProperties); } writeYarnProperties(yarnProps, yarnPropertiesFile); }
appId.toString(), localSrcPath, targetHomeDir,
public static Path getRemotePath(XLearningConfiguration conf, ApplicationId appId, String fileName) { String pathSuffix = appId.toString() + "/" + fileName; Path remotePath = new Path(conf.get(XLearningConfiguration.XLEARNING_STAGING_DIR, XLearningConfiguration.DEFAULT_XLEARNING_STAGING_DIR), pathSuffix); remotePath = new Path(conf.get("fs.defaultFS"), remotePath); LOG.debug("Got remote path of " + fileName + " is " + remotePath.toString()); return remotePath; }
private void addContainerLocalResources(ApplicationId applicationId) throws IOException { Path appWorkDir = GobblinClusterUtils.getAppWorkDirPath(this.fs, this.applicationName, applicationId.toString()); Path containerWorkDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.CONTAINER_WORK_DIR_NAME); if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_JARS_KEY)) { Path appJarsDestDir = new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_JARS_DIR_NAME); addAppJars(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_JARS_KEY), Optional.<Map<String, LocalResource>>absent(), appJarsDestDir); } if (this.config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_FILES_LOCAL_KEY)) { Path appFilesDestDir = new Path(containerWorkDir, GobblinYarnConfigurationKeys.APP_FILES_DIR_NAME); addAppLocalFiles(this.config.getString(GobblinYarnConfigurationKeys.CONTAINER_FILES_LOCAL_KEY), Optional.<Map<String, LocalResource>>absent(), appFilesDestDir); } }
public void killApplication() throws YarnClientException { try { yarnClient.killApplication(appId); } catch (YarnException | IOException e) { throw new YarnClientException( "Kill failed for application: " + appId.toString()); } } }
private static String getApplicationId(ContainerId containerId) { return containerId.getApplicationAttemptId().getApplicationId().toString(); }
private ServiceRecord setupServiceRecord() { ServiceRecord application = new ServiceRecord(); application.set(YarnRegistryAttributes.YARN_ID, jstormMasterContext.appAttemptID.getApplicationId().toString()); application.description = JOYConstants.AM; application.set(YarnRegistryAttributes.YARN_PERSISTENCE, PersistencePolicies.PERMANENT); Map<String, String> addresses = new HashMap<String, String>(); addresses.put(JOYConstants.HOST, jstormMasterContext.appMasterHostname); addresses.put(JOYConstants.PORT, String.valueOf(jstormMasterContext.appMasterThriftPort)); Endpoint endpoint = new Endpoint(JOYConstants.HTTP, JOYConstants.HOST_PORT, JOYConstants.RPC, addresses); application.addExternalEndpoint(endpoint); return application; }
private void createApp() throws ClientException { try { appResponse = client.createAppMaster(); } catch (YarnClientException e) { throw new ClientException("Failed to allocate Drill application master", e); } appId = appResponse.getApplicationId(); System.out.println("Application ID: " + appId.toString()); }
zooKeeperNamespace = commandLine.getOptionValue(zookeeperNamespace.getOpt()); } else { zooKeeperNamespace = effectiveConfiguration.getString(HA_CLUSTER_ID, applicationId.toString());
private static Token<JobTokenIdentifier> createJobToken(ApplicationId applicationId) { String tokenIdentifier = applicationId.toString(); JobTokenIdentifier identifier = new JobTokenIdentifier(new Text( tokenIdentifier)); Token<JobTokenIdentifier> sessionToken = new Token<JobTokenIdentifier>(identifier, new JobTokenSecretManager()); sessionToken.setService(identifier.getJobId()); return sessionToken; } }
private static Token<JobTokenIdentifier> createAmsToken(ApplicationId id) { if (!UserGroupInformation.isSecurityEnabled()) return null; JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(id.toString())); JobTokenSecretManager jobTokenManager = new JobTokenSecretManager(); Token<JobTokenIdentifier> sessionToken = new Token<>(identifier, jobTokenManager); sessionToken.setService(identifier.getJobId()); return sessionToken; }
@Override public void run() throws ClientException { YarnRMClient client = getClient(); System.out.println("Application ID: " + client.getAppId().toString()); Reporter reporter = new Reporter(client); try { reporter.getReport(); } catch (Exception e) { removeAppIdFile(); System.out.println("Application is not running."); return; } reporter.display(opts.verbose, false); if (reporter.isRunning()) { showAmStatus(reporter.report); } }
public GobblinApplicationMaster(String applicationName, ContainerId containerId, Config config, YarnConfiguration yarnConfiguration) throws Exception { super(applicationName, containerId.getApplicationAttemptId().getApplicationId().toString(), config, Optional.<Path>absent()); GobblinYarnLogSource gobblinYarnLogSource = new GobblinYarnLogSource(); if (gobblinYarnLogSource.isLogSourcePresent()) { this.applicationLauncher .addService(gobblinYarnLogSource.buildLogCopier(config, containerId, this.fs, this.appWorkDir)); } this.applicationLauncher .addService(buildYarnService(config, applicationName, this.applicationId, yarnConfiguration, this.fs)); if (UserGroupInformation.isSecurityEnabled()) { LOGGER.info("Adding YarnContainerSecurityManager since security is enabled"); this.applicationLauncher.addService(buildYarnContainerSecurityManager(config, this.fs)); } }
@Test public void testResumeFromYarnID() throws Exception { final Configuration configuration = new Configuration(); final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( configuration, tmp.getRoot().getAbsolutePath(), "y", "yarn"); final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true); final ApplicationId clusterId = flinkYarnSessionCli.getClusterId(commandLine); assertEquals(TEST_YARN_APPLICATION_ID, clusterId); }
@Test public void testResumeFromYarnIDZookeeperNamespace() throws Exception { final Configuration configuration = new Configuration(); final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( configuration, tmp.getRoot().getAbsolutePath(), "y", "yarn"); final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true); final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine); final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration(); String zkNs = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID); assertTrue(zkNs.matches("application_\\d+_0042")); }
@Test public void testYarnIDOverridesPropertiesFile() throws Exception { File directoryPath = writeYarnPropertiesFile(validPropertiesFile); final Configuration configuration = new Configuration(); configuration.setString(YarnConfigOptions.PROPERTIES_FILE_LOCATION, directoryPath.getAbsolutePath()); final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( configuration, tmp.getRoot().getAbsolutePath(), "y", "yarn"); final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID_2.toString() }, true); final ApplicationId clusterId = flinkYarnSessionCli.getClusterId(commandLine); assertEquals(TEST_YARN_APPLICATION_ID_2, clusterId); }
@Test public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception { final Configuration configuration = new Configuration(); final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( configuration, tmp.getRoot().getAbsolutePath(), "y", "yarn"); final String overrideZkNamespace = "my_cluster"; final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true); final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine); final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration(); final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID); assertEquals(overrideZkNamespace, clusterId); }
private SubmitWorkRequestProto createRequest(int fragmentNumber, int numSelfAndUpstreamTasks, int numSelfAndUpstreamComplete, int dagStartTime, int attemptStartTime, int withinDagPriority, String dagName) throws IOException { ApplicationId appId = ApplicationId.newInstance(9999, 72); TezDAGID dagId = TezDAGID.getInstance(appId, 1); TezVertexID vId = TezVertexID.getInstance(dagId, 35); return LlapDaemonTestUtils.buildSubmitProtoRequest(fragmentNumber, appId.toString(), dagId.getId(), vId.getId(), dagName, dagStartTime, attemptStartTime, numSelfAndUpstreamTasks, numSelfAndUpstreamComplete, withinDagPriority, new Credentials()); }