try { sessionState.getSession().addAppMasterLocalFiles(resourceMap); dagClient = sessionState.getSession().submitDAG(dag); } catch (SessionNotRunning nr) { console.printInfo("Tez session was closed. Reopening..."); console.printInfo("Session re-established."); dagClient = sessionState.getSession().submitDAG(dag); TezSessionPoolManager.getInstance().reopenSession(sessionState, this.conf, inputOutputJars, true); dagClient = sessionState.getSession().submitDAG(dag); } catch (Exception retryException) {
private void closeClient(TezClient client) throws TezException, IOException { try { client.stop(); } catch (SessionNotRunning nr) { // ignore } }
private TezClient startSessionAndContainers(TezClient session, HiveConf conf, Map<String, LocalResource> commonLocalResources, TezConfiguration tezConfig, boolean isOnThread) throws TezException, IOException { session.start(); boolean isSuccessful = false; try { tezConfig, n, commonLocalResources); try { session.preWarm(prewarmVertex); } catch (IOException ie) { if (!isOnThread && ie.getMessage().contains("Interrupted while waiting")) { session.waitTillReady(); } catch (InterruptedException ie) { if (isOnThread) {
TezClient tezSession = TezClient.create("UnionExampleSession", tezConf); tezSession.start(); stagingDir, inputPath, outputPath); tezSession.waitTillReady(); dagClient = tezSession.submitDAG(dag); } finally { fs.delete(stagingDir, true); tezSession.stop();
tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false); TezClient tezSession = TezClient.create("OrderedWordCountSession", tezConf, null, instance.credentials); tezSession.start(); if (tezSession.getAppMasterApplicationId() != null) { TezUtilsInternal.setHadoopCallerContext(hadoopShim, tezSession.getAppMasterApplicationId()); maxDataLengthThroughIPC,exceedDataLimit); String callerType = "TestOrderedWordCount"; String callerId = tezSession.getAppMasterApplicationId() == null ? ( "UnknownApp_" + System.currentTimeMillis() + dagIndex ) : ( tezSession.getAppMasterApplicationId().toString() + "_" + dagIndex); dag.setCallerContext(CallerContext.create("Tez", callerId, callerType, "TestOrderedWordCount Job")); tezSession.preWarm(preWarmVertex); waitForTezSessionReady(tezSession); LOG.info("Submitting DAG to Tez Session, dagIndex=" + dagIndex); dagClient = tezSession.submitDAG(dag); LOG.info("Submitted DAG to Tez Session, dagIndex=" + dagIndex); } else { LOG.info("Submitting DAG as a new Tez Application"); dagClient = tezSession.submitDAG(dag); tezSession.stop();
protected void internalNonBlockingStart() throws IOException { try { if( !isTimelineServiceEnabled( jobConfiguration ) ) flowStep.logWarn( "'" + YarnConfiguration.TIMELINE_SERVICE_ENABLED + "' is disabled, please enable to capture detailed metrics of completed flows, this may require starting the YARN timeline server daemon" ); TezConfiguration workingConf = new TezConfiguration( jobConfiguration ); // this could be problematic flowStep.logInfo( "tez session mode enabled: " + workingConf.getBoolean( TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT ) ); prepareEnsureStagingDir( workingConf ); tezClient = TezClient.create( flowStep.getName(), workingConf, ( (Hadoop2TezFlowStep) flowStep ).getAllLocalResources(), null ); tezClient.start(); dagClient = tezClient.submitDAG( dag ); dagId = Util.returnInstanceFieldIfExistsSafe( dagClient, "dagId" ); flowStep.logInfo( "submitted tez dag to app master: {}, with dag id: {}", tezClient.getAppMasterApplicationId(), dagId ); } catch( TezException exception ) { this.throwable = exception; throw new CascadingException( exception ); } }
private static SessionInfo createSession(TezConfiguration amConf, Map<String, LocalResource> requestedAMResources, Credentials creds, TezJobConfig tezJobConf) throws TezException, IOException, InterruptedException { MRToTezHelper.translateMRSettingsForTezAM(amConf); TezScriptState ss = TezScriptState.get(); ss.addDAGSettingsToConf(amConf); if (amConf.getBoolean(PigConfiguration.PIG_TEZ_CONFIGURE_AM_MEMORY, true)) { adjustAMConfig(amConf, tezJobConf); } String jobName = amConf.get(PigContext.JOB_NAME, "pig"); TezClient tezClient = TezClient.create(jobName, amConf, true, requestedAMResources, creds); try { tezClient.start(); TezAppMasterStatus appMasterStatus = tezClient.getAppMasterStatus(); if (appMasterStatus.equals(TezAppMasterStatus.SHUTDOWN)) { throw new RuntimeException("TezSession has already shutdown"); } tezClient.waitTillReady(); } catch (Throwable e) { log.error("Exception while waiting for Tez client to be ready", e); tezClient.stop(); throw new RuntimeException(e); } return new SessionInfo(tezClient, amConf, requestedAMResources); }
session = TezClient.create("HIVE-" + sessionId, tezConfig, true, commonLocalResources, null); session.start(); commonLocalResources); try { session.preWarm(prewarmVertex); } catch (IOException ie) { if (ie.getMessage().contains("Interrupted while waiting")) { session.waitTillReady(); } catch(InterruptedException ie) {
tezClientInternal.waitTillReady(); ApplicationId appId = tezClientInternal.getAppMasterApplicationId(); if (hadoopShim == null) { Configuration conf = (getConf() == null ? new Configuration(false) : getConf()); DAGClient dagClient = tezClientInternal.submitDAG(dag); Set<StatusGetOpts> getOpts = Sets.newHashSet(); if (printCounters) {
String timeStamp = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime()); if (session.getAppMasterStatus().equals( TezAppMasterStatus.SHUTDOWN)) { log.info("Tez session is already shutdown " System.err.println(timeStamp + " Tez session is already shutdown " + session + ", sessionName=" + session.getClientName() + ", applicationId=" + session.getAppMasterApplicationId()); continue; + ", sessionName=" + session.getClientName() + ", applicationId=" + session.getAppMasterApplicationId()); session.stop(); } catch (Exception e) { log.error("Error shutting down Tez session "
.format(Calendar.getInstance().getTime()); System.err.println(timeStamp + " Shutting down Tez session " + ", sessionName=" + session.getClientName() + ", applicationId=" + session.getAppMasterApplicationId()); session.stop(); sessionToRemove = sessionInfo; break;
verifySessionStateForSubmission(); isReady = waitTillReady(timeout, unit); } catch (InterruptedException e) { throw new IOException("Interrupted while waiting for AM to become " + submitDAG(dag); } else { throw new SessionNotReady("Tez AM not ready, could not submit DAG");
tezClient.waitTillReady(); DAGClient dagClient = tezClient.submitDAG(dag); Set<StatusGetOpts> getOpts = Sets.newHashSet(); if (isCountersLog()) {
dag.getCredentials(), tezJobConf); log.info("Submitting DAG " + dag.getName()); dagClient = tezClient.submitDAG(dag); appId = tezClient.getAppMasterApplicationId(); log.info("Submitted DAG " + dag.getName() + ". Application id: " + appId); } catch (Exception e) { if (tezClient != null) { log.error("Cannot submit DAG - Application id: " + tezClient.getAppMasterApplicationId(), e); } else { log.error("Cannot submit DAG", e);
try { dagClient = sessionState.getSession().submitDAG(dag); } catch (SessionNotRunning nr) { console.printInfo("Tez session was closed. Reopening..."); sessionStateRef.value = sessionState = getNewTezSessionOnError(sessionState); console.printInfo("Session re-established."); dagClient = sessionState.getSession().submitDAG(dag); sessionStateRef.value = null; sessionStateRef.value = sessionState = getNewTezSessionOnError(sessionState); dagClient = sessionState.getSession().submitDAG(dag); } catch (Exception retryException) {
"In non-session mode please specify all resources in the initial configuration"); verifySessionStateForSubmission(); Map<String, LocalResource> tezJarResources = getTezJarResources(sessionCredentials); DAGPlan dagPlan = TezClientUtils.prepareAndCreateDAGPlan(dag, amConfig, tezJarResources, usingTezArchiveDeploy, sessionCredentials, servicePluginsDescriptor, javaOptsChecker); proxy = waitForProxy(); } catch (InterruptedException e) { throw new IOException("Interrupted while trying to create a connection to the AM", e); try { LOG.warn("DAG submission to session timed out, stopping session"); stop(); } catch (Throwable t) { LOG.info("Got an exception when trying to stop session", t);
synchronized (sessionInfo) { TezAppMasterStatus appMasterStatus = sessionInfo.session .getAppMasterStatus(); if (appMasterStatus.equals(TezAppMasterStatus.SHUTDOWN)) { sessionsToRemove.add(sessionInfo); if (shutdown == true) { log.info("Shutting down Tez session " + newSession.session); newSession.session.stop(); throw new IOException("TezSessionManager is shut down");
@Private @VisibleForTesting public ApplicationId getAppId() { if (tezClientInternal == null) { LOG.warn("TezClient is not initialized, return null for AppId"); return null; } return tezClientInternal.getAppMasterApplicationId(); } }
private static void waitForTezSessionReady(TezClient tezSession) throws IOException, TezException, InterruptedException { tezSession.waitTillReady(); }