public JstormOnYarn() throws Exception { this(new YarnConfiguration()); }
/** * Return standard class path entries from the YARN application class path. */ public String[] getYarnAppClassPath() { return conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH); }
public static String extractJobHistoryUrl(String yarnWebapp, Configuration conf) { Pattern pattern = Pattern.compile("(http(s)?://)([^:]*):([^/])*.*"); Matcher m = pattern.matcher(yarnWebapp); Preconditions.checkArgument(m.matches(), "Yarn master URL" + yarnWebapp + " not right."); return m.group(1) + HAUtil.getConfValueForRMInstance("mapreduce.jobhistory.webapp.address", m.group(2) + ":19888", conf); } }
@Override public void run() { // Dump configuration if requested for diagnostic use. System.out.println("----------------------------------------------"); System.out.println("Effective Drill-on-YARN Configuration"); DrillOnYarnConfig.instance().dump(); System.out.println("----------------------------------------------"); // Dump YARN configuration. System.out.println("YARN, DFS and Hadoop Configuration"); YarnConfiguration conf = new YarnConfiguration(); try { YarnConfiguration.dumpConfiguration(conf, new OutputStreamWriter(System.out)); System.out.println(); } catch (IOException e) { // Ignore; } System.out.println("----------------------------------------------"); } }
@BeforeClass public static void setup() { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 768); YARN_CONFIGURATION.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-fifo"); startYARNWithConfig(YARN_CONFIGURATION); }
@BeforeClass public static void setup() { YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-per-job"); startYARNWithConfig(YARN_CONFIGURATION); }
@Override public boolean apply(Void input) { return !clusterConf.get(YarnConfiguration.RM_ADDRESS).contains(":0"); } }, "Waiting for RM");
public AbstractYarnClusterDescriptor( Configuration flinkConfiguration, YarnConfiguration yarnConfiguration, String configurationDirectory, YarnClient yarnClient, boolean sharedYarnClient) { this.yarnConfiguration = Preconditions.checkNotNull(yarnConfiguration); // for unit tests only if (System.getenv("IN_TESTS") != null) { try { yarnConfiguration.addResource(new File(System.getenv("YARN_CONF_DIR"), "yarn-site.xml").toURI().toURL()); } catch (Throwable t) { throw new RuntimeException("Error", t); } } this.yarnClient = Preconditions.checkNotNull(yarnClient); this.sharedYarnClient = sharedYarnClient; this.flinkConfiguration = Preconditions.checkNotNull(flinkConfiguration); userJarInclusion = getUserJarInclusionMode(flinkConfiguration); this.configurationDirectory = Preconditions.checkNotNull(configurationDirectory); }
public YarnRMClient() { this(new YarnConfiguration()); }
@BeforeClass public static void setup() throws Exception { zkServer = new TestingServer(); storageDir = FOLDER.newFolder().getAbsolutePath(); // startYARNWithConfig should be implemented by subclass YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, LOG_DIR); YARN_CONFIGURATION.setInt(YarnConfiguration.NM_PMEM_MB, 4096); startYARNWithConfig(YARN_CONFIGURATION); }
public static boolean verifyStringsInNamedLogFiles( final String[] mustHave, final String fileName) { List<String> mustHaveList = Arrays.asList(mustHave); File cwd = new File("target/" + YARN_CONFIGURATION.get(TEST_CLUSTER_NAME_KEY)); if (!cwd.exists() || !cwd.isDirectory()) { return false;
/** * @param envs Environment variables. * @param conf Yarn configuration. */ private static void setupAppMasterEnv(Map<String, String> envs, YarnConfiguration conf) { for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) Apps.addToEnvironment(envs, Environment.CLASSPATH.name(), c.trim(), File.pathSeparator); Apps.addToEnvironment(envs, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*", File.pathSeparator); } }
/** * @param ignitePath Hdfs path to ignite. * @param props Cluster properties. */ public ApplicationMaster(String ignitePath, ClusterProperties props) throws Exception { this.conf = new YarnConfiguration(); this.props = props; this.ignitePath = new Path(ignitePath); }
@BeforeClass public static void setup() throws Exception { YARN_CONFIGURATION.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class); YARN_CONFIGURATION.set("yarn.scheduler.capacity.root.queues", "default,qa-team"); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.default.capacity", 40); YARN_CONFIGURATION.setInt("yarn.scheduler.capacity.root.qa-team.capacity", 60); YARN_CONFIGURATION.set(YarnTestBase.TEST_CLUSTER_NAME_KEY, "flink-yarn-tests-capacityscheduler"); startYARNWithConfig(YARN_CONFIGURATION); restClientExecutor = Executors.newSingleThreadExecutor(); restClient = new RestClient(RestClientConfiguration.fromConfiguration(new Configuration()), restClientExecutor); }
public JstormMaster() { // Set up the configuration conf = new YarnConfiguration(); Path jstormyarnConfPath = new Path("jstorm-yarn.xml"); conf.addResource(jstormyarnConfPath); }
/** * Create a configuration with Slider-specific tuning. * This is done rather than doing custom configs. * * @return the config */ public static YarnConfiguration createConfiguration() { YarnConfiguration conf = new YarnConfiguration(); patchConfiguration(conf); return conf; }
public AzkabanGobblinYarnAppLauncher(String jobId, Properties props) throws IOException { super(jobId, LOGGER); Config gobblinConfig = ConfigUtils.propertiesToConfig(props); this.gobblinYarnAppLauncher = new GobblinYarnAppLauncher(gobblinConfig, new YarnConfiguration()); }
public static void main(String[] args) throws Exception { final GobblinYarnAppLauncher gobblinYarnAppLauncher = new GobblinYarnAppLauncher(ConfigFactory.load(), new YarnConfiguration()); Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { try { gobblinYarnAppLauncher.stop(); } catch (IOException ioe) { LOGGER.error("Failed to shutdown the " + GobblinYarnAppLauncher.class.getSimpleName(), ioe); } catch (TimeoutException te) { LOGGER.error("Timeout in stopping the service manager", te); } finally { if (gobblinYarnAppLauncher.emailNotificationOnShutdown) { gobblinYarnAppLauncher.sendEmailOnShutdown(Optional.<ApplicationReport>absent()); } } } }); gobblinYarnAppLauncher.launch(); } }
/** * Lazy loading of YARN configuration since it takes a long time to load. * (YARN provides no caching, sadly.) */ private void loadYarnConfig() { if (yarnConf == null) { yarnConf = new YarnConfiguration(); // On some distributions, lack of proper configuration causes // DFS to default to the local file system. So, a local file // system generally means that the config is wrong, or running // the wrong build of Drill for the user's environment. URI fsUri = FileSystem.getDefaultUri( yarnConf ); if(fsUri.toString().startsWith("file:/")) { System.err.println("Warning: Default DFS URI is for a local file system: " + fsUri); } } }
@BeforeClass public static void setupClass() { yarnConfiguration = new YarnConfiguration(); yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConfiguration); yarnClient.start(); }