/** * The entry point for the YARN task executor runner. * * @param args The command line arguments. */ public static void main(String[] args) { EnvironmentInformation.logEnvironmentInfo(LOG, "YARN TaskExecutor runner", args); SignalHandler.register(LOG); JvmShutdownSafeguard.installAsShutdownHook(LOG); run(args); }
@Override public TaskManagerRuntimeInfo getTaskManagerInfo() { return new TestingTaskManagerRuntimeInfo(); }
@Override public LeaderConnectionInfo getClusterConnectionInfo() throws LeaderRetrievalException { return LeaderRetrievalUtils.retrieveLeaderConnectionInfo( highAvailabilityServices.getDispatcherLeaderRetriever(), timeout); }
/** * Helper method to wait until the {@link Dispatcher} has set its fencing token. * * @param rpcService to use to connect to the dispatcher * @param haServices high availability services to connect to the dispatcher * @return {@link DispatcherGateway} * @throws Exception if something goes wrong */ static DispatcherGateway retrieveDispatcherGateway(RpcService rpcService, HighAvailabilityServices haServices) throws Exception { final LeaderConnectionInfo leaderConnectionInfo = LeaderRetrievalUtils.retrieveLeaderConnectionInfo(haServices.getDispatcherLeaderRetriever(), Time.seconds(10L)); return rpcService.connect( leaderConnectionInfo.getAddress(), DispatcherId.fromUuid(leaderConnectionInfo.getLeaderSessionID()), DispatcherGateway.class).get(); }
@Test public void loadFromClasspathByDefault() { org.apache.hadoop.conf.Configuration hadoopConf = HadoopUtils.getHadoopConfiguration(new Configuration()); assertEquals(IN_CP_CONFIG_VALUE, hadoopConf.get(IN_CP_CONFIG_KEY, null)); }
public <T extends StreamOperator<?>> T getStreamOperator(ClassLoader cl) { try { return InstantiationUtil.readObjectFromConfig(this.config, SERIALIZEDUDF, cl); } catch (ClassNotFoundException e) { String classLoaderInfo = ClassLoaderUtil.getUserCodeClassLoaderInfo(cl); boolean loadableDoubleCheck = ClassLoaderUtil.validateClassLoadable(e, cl); String exceptionMessage = "Cannot load user class: " + e.getMessage() + "\nClassLoader info: " + classLoaderInfo + (loadableDoubleCheck ? "\nClass was actually found in classloader - deserialization issue." : "\nClass not resolvable through given classloader."); throw new StreamTaskException(exceptionMessage, e); } catch (Exception e) { throw new StreamTaskException("Cannot instantiate user function.", e); } }
@Test public void testUnresolvableHostname1() throws UnknownHostException, ConfigurationException { Configuration config = new Configuration(); config.setString(JobManagerOptions.ADDRESS, nonExistingHostname); config.setInteger(JobManagerOptions.PORT, 17234); StandaloneUtils.createLeaderRetrievalService( config, false, JobMaster.JOB_MANAGER_NAME); }
@SuppressWarnings("WeakerAccess") public SerializingLongReceiver(InputGate inputGate, int expectedRepetitionsOfExpectedRecord) { super(expectedRepetitionsOfExpectedRecord); this.reader = new MutableRecordReader<>( inputGate, new String[]{ EnvironmentInformation.getTemporaryFileDirectory() }); }
@Override public MesosWorkerStore createMesosWorkerStore(Configuration configuration, Executor executor) throws Exception { RetrievableStateStorageHelper<MesosWorkerStore.Worker> stateStorageHelper = ZooKeeperUtils.createFileSystemStateStorage(configuration, "mesosWorkerStore"); ZooKeeperStateHandleStore<MesosWorkerStore.Worker> zooKeeperStateHandleStore = zooKeeperUtilityFactory.createZooKeeperStateHandleStore( "/workers", stateStorageHelper); ZooKeeperSharedValue frameworkId = zooKeeperUtilityFactory.createSharedValue("/frameworkId", new byte[0]); ZooKeeperSharedCount totalTaskCount = zooKeeperUtilityFactory.createSharedCount("/taskCount", 0); return new ZooKeeperMesosWorkerStore( zooKeeperStateHandleStore, frameworkId, totalTaskCount); }
/** * Creates a new Recoverable writer. * @param fs The Hadoop file system on which the writer operates. */ public HadoopRecoverableWriter(org.apache.hadoop.fs.FileSystem fs) { this.fs = checkNotNull(fs); // This writer is only supported on a subset of file systems, and on // specific versions. We check these schemes and versions eagerly for // better error messages. if (!"hdfs".equalsIgnoreCase(fs.getScheme()) || !HadoopUtils.isMinHadoopVersion(2, 7)) { throw new UnsupportedOperationException( "Recoverable writers on Hadoop are only supported for HDFS and for Hadoop version 2.7 or newer"); } }
@After public void teardown() throws Exception { if (testingFatalErrorHandler != null) { testingFatalErrorHandler.rethrowError(); } if (env != null) { env.clear(); } }
/** * The entry point for the YARN application master. * * @param args The command line arguments. */ public static void main(String[] args) { EnvironmentInformation.logEnvironmentInfo(LOG, "YARN ApplicationMaster / ResourceManager / JobManager", args); SignalHandler.register(LOG); JvmShutdownSafeguard.installAsShutdownHook(LOG); // run and exit with the proper return code int returnCode = new YarnApplicationMasterRunner().run(args); System.exit(returnCode); }
@Override public LeaderConnectionInfo getClusterConnectionInfo() throws LeaderRetrievalException { return LeaderRetrievalUtils.retrieveLeaderConnectionInfo( highAvailabilityServices.getDispatcherLeaderRetriever(), timeout); }
@Test public void testUnresolvableHostname2() throws Exception { try { Configuration config = new Configuration(); config.setString(JobManagerOptions.ADDRESS, nonExistingHostname); config.setInteger(JobManagerOptions.PORT, 17234); StandaloneUtils.createLeaderRetrievalService( config, true, JobMaster.JOB_MANAGER_NAME); fail("This should fail with an UnknownHostException"); } catch (UnknownHostException e) { // that is what we want! } }
@BeforeClass public static void testHadoopVersion() { Assume.assumeTrue(HadoopUtils.isMinHadoopVersion(2, 7)); }
public static void main(String[] args) { EnvironmentInformation.logEnvironmentInfo(LOG, "YARN ApplicationMaster / JobManager", args); SignalHandler.register(LOG); JvmShutdownSafeguard.installAsShutdownHook(LOG); // run and exit with the proper return code int returnCode = new TestingApplicationMaster().run(args); System.exit(returnCode); }
/** * Gets the current cluster connection info (may change in case of a HA setup). * * @return The the connection info to the leader component of the cluster * @throws LeaderRetrievalException if the leader could not be retrieved */ public LeaderConnectionInfo getClusterConnectionInfo() throws LeaderRetrievalException { return LeaderRetrievalUtils.retrieveLeaderConnectionInfo( highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID), timeout); }
public static void main(String[] args) { // startup checks and logging EnvironmentInformation.logEnvironmentInfo(LOG, MesosSessionClusterEntrypoint.class.getSimpleName(), args); SignalHandler.register(LOG); JvmShutdownSafeguard.installAsShutdownHook(LOG); // load configuration incl. dynamic properties CommandLineParser parser = new PosixParser(); CommandLine cmd; try { cmd = parser.parse(ALL_OPTIONS, args); } catch (Exception e){ LOG.error("Could not parse the command-line options.", e); System.exit(STARTUP_FAILURE_RETURN_CODE); return; } Configuration dynamicProperties = BootstrapTools.parseDynamicProperties(cmd); Configuration configuration = MesosEntrypointUtils.loadConfiguration(dynamicProperties, LOG); MesosSessionClusterEntrypoint clusterEntrypoint = new MesosSessionClusterEntrypoint(configuration, dynamicProperties); ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint); }
public static void main(String[] args) { // startup checks and logging EnvironmentInformation.logEnvironmentInfo(LOG, MesosJobClusterEntrypoint.class.getSimpleName(), args); SignalHandler.register(LOG); JvmShutdownSafeguard.installAsShutdownHook(LOG); // load configuration incl. dynamic properties CommandLineParser parser = new PosixParser(); CommandLine cmd; try { cmd = parser.parse(ALL_OPTIONS, args); } catch (Exception e){ LOG.error("Could not parse the command-line options.", e); System.exit(STARTUP_FAILURE_RETURN_CODE); return; } Configuration dynamicProperties = BootstrapTools.parseDynamicProperties(cmd); Configuration configuration = MesosEntrypointUtils.loadConfiguration(dynamicProperties, LOG); MesosJobClusterEntrypoint clusterEntrypoint = new MesosJobClusterEntrypoint(configuration, dynamicProperties); ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint); } }
public static void main(String[] args) { // startup checks and logging EnvironmentInformation.logEnvironmentInfo(LOG, YarnJobClusterEntrypoint.class.getSimpleName(), args); SignalHandler.register(LOG); JvmShutdownSafeguard.installAsShutdownHook(LOG); Map<String, String> env = System.getenv(); final String workingDirectory = env.get(ApplicationConstants.Environment.PWD.key()); Preconditions.checkArgument( workingDirectory != null, "Working directory variable (%s) not set", ApplicationConstants.Environment.PWD.key()); try { YarnEntrypointUtils.logYarnEnvironmentInformation(env, LOG); } catch (IOException e) { LOG.warn("Could not log YARN environment information.", e); } Configuration configuration = YarnEntrypointUtils.loadConfiguration(workingDirectory, env, LOG); YarnJobClusterEntrypoint yarnJobClusterEntrypoint = new YarnJobClusterEntrypoint( configuration, workingDirectory); ClusterEntrypoint.runClusterEntrypoint(yarnJobClusterEntrypoint); } }