public AbstractYarnClusterDescriptor( Configuration flinkConfiguration, YarnConfiguration yarnConfiguration, String configurationDirectory, YarnClient yarnClient, boolean sharedYarnClient) { this.yarnConfiguration = Preconditions.checkNotNull(yarnConfiguration); // for unit tests only if (System.getenv("IN_TESTS") != null) { try { yarnConfiguration.addResource(new File(System.getenv("YARN_CONF_DIR"), "yarn-site.xml").toURI().toURL()); } catch (Throwable t) { throw new RuntimeException("Error", t); } } this.yarnClient = Preconditions.checkNotNull(yarnClient); this.sharedYarnClient = sharedYarnClient; this.flinkConfiguration = Preconditions.checkNotNull(flinkConfiguration); userJarInclusion = getUserJarInclusionMode(flinkConfiguration); this.configurationDirectory = Preconditions.checkNotNull(configurationDirectory); }
public Executor(String instancName, String shellCommand, STARTType startType, String runningContainer, String localDir, String deployPath, String hadoopHome, String javaHome, String pythonHome, String dstPath, String portList, String shellArgs, String ExecShellStringPath, String applicationId, String supervisorLogviewPort, String nimbusThriftPort) { executorMeta = new ExecutorMeta(instancName, shellCommand, startType, runningContainer, localDir, deployPath, hadoopHome, javaHome, pythonHome, dstPath, portList, shellArgs, ExecShellStringPath, applicationId, supervisorLogviewPort, nimbusThriftPort); conf = new YarnConfiguration(); Path yarnSite = new Path(hadoopHome + JOYConstants.YARN_SITE_PATH); conf.addResource(yarnSite); //get first log dir logDir = conf.get(JOYConstants.YARN_NM_LOG, JOYConstants.YARN_NM_LOG_DIR).split(JOYConstants.COMMA)[0] + JOYConstants.BACKLASH + applicationId + JOYConstants.BACKLASH + runningContainer; //Setup RegistryOperations registryOperations = RegistryOperationsFactory.createInstance(JOYConstants.YARN_REGISTRY, conf); try { setupInitialRegistryPaths(); } catch (IOException e) { e.printStackTrace(); } registryOperations.start(); }
public YarnClusterConfiguration toYarnClusterConfiguration() { Preconditions.checkNotNull(yarnSite, "yarn.site.location is not configured"); Preconditions.checkNotNull(homeDir, "athenax.home.dir is not configured"); Preconditions.checkNotNull(flinkUberJar, "flink.uber.jar.location is not configured"); YarnConfiguration yarnConf = new YarnConfiguration(); yarnConf.addResource(new Path(URI.create(yarnSite))); return new YarnClusterConfiguration( yarnConf, homeDir, new Path(flinkUberJar), resourcesToLocalize.stream().map(x -> new Path(URI.create(x))).collect(Collectors.toSet()), additionalJars.stream().map(x -> new Path(URI.create(x))).collect(Collectors.toSet())); } }
public static YarnConfiguration getYarnConf(String yarnConfDir) { YarnConfiguration yarnConf = new YarnConfiguration(); try { File dir = new File(yarnConfDir); if(dir.exists() && dir.isDirectory()) { File[] xmlFileList = new File(yarnConfDir).listFiles((dir1, name) -> { if(name.endsWith(".xml")){ return true; } return false; }); if(xmlFileList != null) { for(File xmlFile : xmlFileList) { yarnConf.addResource(xmlFile.toURI().toURL()); } } } } catch(Exception e) { throw new RuntimeException(e); } haYarnConf(yarnConf); return yarnConf; }
public static YarnConfiguration getYarnConf(String yarnConfDir) { YarnConfiguration yarnConf = new YarnConfiguration(); try { File dir = new File(yarnConfDir); if(dir.exists() && dir.isDirectory()) { File[] xmlFileList = new File(yarnConfDir).listFiles((dir1, name) -> { if(name.endsWith(".xml")){ return true; } return false; }); if(xmlFileList != null) { for(File xmlFile : xmlFileList) { yarnConf.addResource(xmlFile.toURI().toURL()); } } } } catch(Exception e) { throw new RuntimeException(e); } haYarnConf(yarnConf); return yarnConf; }
public static YarnConfiguration getYarnConfiguration(Configuration config, String yarnConfDir) throws Exception { YarnConfiguration yarnConf = new YarnConfiguration(); config.setString(ConfigConstants.PATH_HADOOP_CONFIG, yarnConfDir); FileSystem.initialize(config); File dir = new File(yarnConfDir); if (dir.exists() && dir.isDirectory()) { File[] xmlFileList = new File(yarnConfDir).listFiles(new FilenameFilter() { @Override public boolean accept(File dir, String name) { if (name.endsWith(".xml")) { return true; } return false; } }); if (xmlFileList != null) { for (File xmlFile : xmlFileList) { yarnConf.addResource(xmlFile.toURI().toURL()); } } } return yarnConf; }
private void createYarnClient() { if (this.yarnConfAddress != null) { this.yarnConf.addResource(new Path(this.yarnConfAddress)); } if (this.hdfsConfAddress != null) { this.hdfsConf.addResource(new Path(this.hdfsConfAddress)); } int numRMConnectRetries = tonyConf.getInt(TonyConfigurationKeys.RM_CLIENT_CONNECT_RETRY_MULTIPLIER, TonyConfigurationKeys.DEFAULT_RM_CLIENT_CONNECT_RETRY_MULTIPLIER); long rmMaxWaitMS = yarnConf.getLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS) * numRMConnectRetries; yarnConf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, rmMaxWaitMS); if (System.getenv(Constants.HADOOP_CONF_DIR) != null) { hdfsConf.addResource(new Path(System.getenv(Constants.HADOOP_CONF_DIR) + File.separatorChar + Constants.CORE_SITE_CONF)); yarnConf.addResource(new Path(System.getenv(Constants.HADOOP_CONF_DIR) + File.separatorChar + Constants.CORE_SITE_CONF)); hdfsConf.addResource(new Path(System.getenv(Constants.HADOOP_CONF_DIR) + File.separatorChar + Constants.HDFS_SITE_CONF)); } yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConf); }
private synchronized TwillRunnerService getTwillRunner(FluoConfiguration config) { if (!twillRunners.containsKey(config.getApplicationName())) { YarnConfiguration yarnConfig = new YarnConfiguration(); yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/core-site.xml")); yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/yarn-site.xml")); TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, config.getAppZookeepers() + ZookeeperPath.TWILL); twillRunner.start(); twillRunners.put(config.getApplicationName(), twillRunner); // sleep to give twill time to retrieve state from zookeeper try { Thread.sleep(1000); } catch (InterruptedException e) { throw new IllegalStateException(e); } } return twillRunners.get(config.getApplicationName()); }
private synchronized TwillRunnerService getTwillRunner(FluoConfiguration config) { if (!twillRunners.containsKey(config.getApplicationName())) { YarnConfiguration yarnConfig = new YarnConfiguration(); yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/core-site.xml")); yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/yarn-site.xml")); TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, config.getAppZookeepers() + ZookeeperPath.TWILL); twillRunner.start(); twillRunners.put(config.getApplicationName(), twillRunner); // sleep to give twill time to retrieve state from zookeeper try { Thread.sleep(1000); } catch (InterruptedException e) { throw new IllegalStateException(e); } } return twillRunners.get(config.getApplicationName()); }
private synchronized TwillRunnerService getTwillRunner(FluoConfiguration config) { if (!twillRunners.containsKey(config.getApplicationName())) { YarnConfiguration yarnConfig = new YarnConfiguration(); yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/core-site.xml")); yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/yarn-site.xml")); TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfig, config.getAppZookeepers() + ZookeeperPath.TWILL); twillRunner.start(); twillRunners.put(config.getApplicationName(), twillRunner); // sleep to give twill time to retrieve state from zookeeper try { Thread.sleep(1000); } catch (InterruptedException e) { throw new IllegalStateException(e); } } return twillRunners.get(config.getApplicationName()); }
public AbstractYarnClusterDescriptor( Configuration flinkConfiguration, YarnConfiguration yarnConfiguration, String configurationDirectory, YarnClient yarnClient, boolean sharedYarnClient) { this.yarnConfiguration = Preconditions.checkNotNull(yarnConfiguration); // for unit tests only if (System.getenv("IN_TESTS") != null) { try { yarnConfiguration.addResource(new File(System.getenv("YARN_CONF_DIR"), "yarn-site.xml").toURI().toURL()); } catch (Throwable t) { throw new RuntimeException("Error", t); } } this.yarnClient = Preconditions.checkNotNull(yarnClient); this.sharedYarnClient = sharedYarnClient; this.flinkConfiguration = Preconditions.checkNotNull(flinkConfiguration); userJarInclusion = getUserJarInclusionMode(flinkConfiguration); this.configurationDirectory = Preconditions.checkNotNull(configurationDirectory); }
public AbstractYarnClusterDescriptor( Configuration flinkConfiguration, YarnConfiguration yarnConfiguration, String configurationDirectory, YarnClient yarnClient, boolean sharedYarnClient) { this.yarnConfiguration = Preconditions.checkNotNull(yarnConfiguration); // for unit tests only if (System.getenv("IN_TESTS") != null) { try { yarnConfiguration.addResource(new File(System.getenv("YARN_CONF_DIR"), "yarn-site.xml").toURI().toURL()); } catch (Throwable t) { throw new RuntimeException("Error", t); } } this.yarnClient = Preconditions.checkNotNull(yarnClient); this.sharedYarnClient = sharedYarnClient; this.flinkConfiguration = Preconditions.checkNotNull(flinkConfiguration); userJarInclusion = getUserJarInclusionMode(flinkConfiguration); this.configurationDirectory = Preconditions.checkNotNull(configurationDirectory); }
if (resources != null) { for (Resource resource : resources) { internalConfig.addResource(resource.getURL());