Refine search
@Override public void run() { System.clearProperty(XLearningConstants.Environment.HADOOP_USER_NAME.toString()); YarnConfiguration xlearningConf = new YarnConfiguration(); if (xlearningConf.getBoolean(XLearningConfiguration.XLEARNING_CLEANUP_ENABLE, XLearningConfiguration.DEFAULT_XLEARNING_CLEANUP_ENABLE)) { Path stagingDir = new Path(envs.get(XLearningConstants.Environment.XLEARNING_STAGING_LOCATION.toString())); try { Path logdir = new Path(conf.get(XLearningConfiguration.XLEARNING_HISTORY_LOG_DIR, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_DIR) + "/" + applicationAttemptID.getApplicationId().toString() + "/" + applicationAttemptID.getApplicationId().toString()); Path jobLogPath = new Path(xlearningConf.get("fs.defaultFS"), logdir); LOG.info("jobLogPath:" + jobLogPath.toString()); LOG.info("Start write the log to " + jobLogPath.toString()); FSDataOutputStream out = fs.create(jobLogPath); fs.setPermission(jobLogPath, new FsPermission(LOG_FILE_PERMISSION)); if (conf.getBoolean(XLearningConfiguration.XLEARNING_HOST_LOCAL_ENABLE, XLearningConfiguration.DEFAULT_XLEARNING_HOST_LOCAL_ENABLE)) { Path hostLocaldir = new Path(conf.get(XLearningConfiguration.XLEARNING_HISTORY_LOG_DIR, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_DIR) + "/" + conf.get("hadoop.job.ugi").split(",")[0] + "/" + envs.get(XLearningConstants.Environment.XLEARNING_APP_NAME.toString())); Path hostLocalPath = new Path(xlearningConf.get("fs.defaultFS"), hostLocaldir); try { FSDataOutputStream hostLocalOut = fs.create(hostLocalPath); Path boardLogPath; if (conf.get(XLearningConfiguration.XLEARNING_TF_BOARD_LOG_DIR, XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_LOG_DIR).indexOf("hdfs://") == -1) { if (conf.get(XLearningConfiguration.XLEARNING_TF_BOARD_HISTORY_DIR, XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_HISTORY_DIR).equals(xlearningConf.get(XLearningConfiguration.XLEARNING_TF_BOARD_HISTORY_DIR, XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_HISTORY_DIR))) { boardLogPath = new Path(xlearningConf.get("fs.defaultFS"), conf.get(XLearningConfiguration.XLEARNING_TF_BOARD_HISTORY_DIR, XLearningConfiguration.DEFAULT_XLEARNING_TF_BOARD_HISTORY_DIR) + "/" + applicationAttemptID.getApplicationId().toString());
@BeforeClass public static void setup() { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768); YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo"); startYARNWithConfig(YARN_CONFIGURATION); }
if (YarnConfiguration.useHttps(conf)) { webappConfKey = YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS; defaultAddr = YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS; YarnConfiguration yarnConf = new YarnConfiguration(conf); String active = RMHAUtils.findActiveRMHAId(yarnConf); rmWebHost = HAUtil.getConfValueForRMInstance(HAUtil.addSuffix(webappConfKey, active), defaultAddr, rmWebHost = (YarnConfiguration.useHttps(conf) ? "https://" : "http://") + rmWebHost;
public Executor(String instancName, String shellCommand, STARTType startType, String runningContainer, String localDir, String deployPath, String hadoopHome, String javaHome, String pythonHome, String dstPath, String portList, String shellArgs, String ExecShellStringPath, String applicationId, String supervisorLogviewPort, String nimbusThriftPort) { executorMeta = new ExecutorMeta(instancName, shellCommand, startType, runningContainer, localDir, deployPath, hadoopHome, javaHome, pythonHome, dstPath, portList, shellArgs, ExecShellStringPath, applicationId, supervisorLogviewPort, nimbusThriftPort); conf = new YarnConfiguration(); Path yarnSite = new Path(hadoopHome + JOYConstants.YARN_SITE_PATH); conf.addResource(yarnSite); //get first log dir logDir = conf.get(JOYConstants.YARN_NM_LOG, JOYConstants.YARN_NM_LOG_DIR).split(JOYConstants.COMMA)[0] + JOYConstants.BACKLASH + applicationId + JOYConstants.BACKLASH + runningContainer; //Setup RegistryOperations registryOperations = RegistryOperationsFactory.createInstance(JOYConstants.YARN_REGISTRY, conf); try { setupInitialRegistryPaths(); } catch (IOException e) { e.printStackTrace(); } registryOperations.start(); }
@Override public void run() { // Dump configuration if requested for diagnostic use. System.out.println("----------------------------------------------"); System.out.println("Effective Drill-on-YARN Configuration"); DrillOnYarnConfig.instance().dump(); System.out.println("----------------------------------------------"); // Dump YARN configuration. System.out.println("YARN, DFS and Hadoop Configuration"); YarnConfiguration conf = new YarnConfiguration(); try { YarnConfiguration.dumpConfiguration(conf, new OutputStreamWriter(System.out)); System.out.println(); } catch (IOException e) { // Ignore; } System.out.println("----------------------------------------------"); } }
public static void setRMWebAppHostnameAndPort(Configuration conf, String hostname, int port) { String resolvedAddress = hostname + ":" + port; if (YarnConfiguration.useHttps(conf)) { conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, resolvedAddress); } else { conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resolvedAddress); } }
public static String getAHSWebAppURLWithoutScheme(Configuration conf) { if (YarnConfiguration.useHttps(conf)) { return conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS); } else { return conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS); } }
private static MapReduceContextConfig createContextConfig() { Configuration conf = new Configuration(new YarnConfiguration()); conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE)); return new MapReduceContextConfig(conf); }
public TestApplicationHistoryManagerOnTimelineStore(String caller) { conf = new YarnConfiguration(); if (!caller.equals("")) { callerUGI = UserGroupInformation.createRemoteUser(caller, AuthMethod.SIMPLE); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); } }
@Before public void setUp() throws Exception { conf = new YarnConfiguration(); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); mgr = new NullRMNodeLabelsManager(); mgr.init(conf); }
@Test(timeout = 60000, expected = YarnRuntimeException.class) public void testTimelineCollectorManagerWithInvalidTimelineWriter() { Configuration conf = new YarnConfiguration(); conf.set(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS, Object.class.getName()); runTimelineCollectorManagerWithConfig(conf); }
@Test(timeout = 60000, expected = YarnRuntimeException.class) public void testTimelineReaderServerWithInvalidTimelineReader() { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_WEBAPP_ADDRESS, "localhost:0"); conf.set(YarnConfiguration.TIMELINE_SERVICE_READER_CLASS, Object.class.getName()); runTimelineReaderServerWithConfig(conf); }
private Configuration getConfiguration() { Configuration configuration = new YarnConfiguration(); configuration.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true); configuration.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, "target/logs"); configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true); configuration.set(YarnConfiguration.YARN_ADMIN_ACL, "admin"); return configuration; }
private YarnClusterConfiguration prepareYarnCluster() throws IOException, URISyntaxException { yarnCluster.init(yarnConf); yarnCluster.start(); yarnConf.set(RM_ADDRESS, yarnCluster.getResourceManager().getConfig().get(RM_ADDRESS)); File yarnSite = new File(workDir, "yarn-site.xml"); try (PrintWriter pw = new PrintWriter(new FileWriter(yarnSite))) { yarnConf.writeXml(pw); } Path flinkUberJar = new Path(new File(workDir, "flink.jar").toURI()); Path flinkConfYaml = new Path(new File(workDir, "flink-conf.yaml").toURI()); @SuppressWarnings("ConstantConditions") Path log4jPath = new Path(Thread.currentThread().getContextClassLoader().getResource("log4j.properties").toURI()); Set<Path> resourcesToLocalize = new HashSet<>(Arrays.asList(flinkUberJar, flinkConfYaml, log4jPath)); String home = workDir.toURI().toString(); return new YarnClusterConfiguration( yarnConf, home, flinkUberJar, resourcesToLocalize, systemJars(yarnSite)); }
@Before public void setup() throws UnknownHostException { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); conf = new YarnConfiguration(); UserGroupInformation.setConfiguration(conf); conf.set(YarnConfiguration.RECOVERY_ENABLED, "true"); conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); Assert.assertTrue(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS > 1); }
@Test public void testYarnACLsNotEnabledForDomain() throws Exception { Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false); TimelineACLsManager timelineACLsManager = new TimelineACLsManager(conf); TimelineDomain domain = new TimelineDomain(); domain.setOwner("owner"); Assert.assertTrue( "Always true when ACLs are not enabled", timelineACLsManager.checkAccess( UserGroupInformation.createRemoteUser("user"), domain)); }
public JstormMaster() { // Set up the configuration conf = new YarnConfiguration(); Path jstormyarnConfPath = new Path("jstorm-yarn.xml"); conf.addResource(jstormyarnConfPath); }
@Before public void before() { mgr = new DummyCommonNodeLabelsManager(); Configuration conf = new YarnConfiguration(); conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); mgr.init(conf); mgr.start(); }
public JstormOnYarn() throws Exception { this(new YarnConfiguration()); }
@Before public void setup() { Logger rootLogger = LogManager.getRootLogger(); rootLogger.setLevel(Level.DEBUG); ExitUtil.disableSystemExit(); testConf = new YarnConfiguration(); testConf .set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); UserGroupInformation.setLoginUser(null); UserGroupInformation.setConfiguration(testConf); }