public void open() throws IOException, LoginException, URISyntaxException, TezException { String[] noFiles = null; open(noFiles); }
public void open(HiveConf conf) throws IOException, LoginException, URISyntaxException, TezException { Set<String> noFiles = null; open(conf, noFiles, null); }
static void reopenInternal( TezSessionState sessionState) throws Exception { HiveResources resources = sessionState.extractHiveResources(); // TODO: close basically resets the object to a bunch of nulls. // We should ideally not reuse the object because it's pointless and error-prone. sessionState.close(false); // Note: scratchdir is reused implicitly because the sessionId is the same. sessionState.open(resources); }
/** * Ensures that the Tez Session is open and the AM has all necessary jars configured. */ @VisibleForTesting void ensureSessionHasResources( TezSessionState session, String[] nonConfResources) throws Exception { TezClient client = session.getSession(); // TODO null can also mean that this operation was interrupted. Should we really try to re-create the session in that case ? if (client == null) { // Note: the only sane case where this can happen is the non-pool one. We should get rid // of it, in non-pool case perf doesn't matter so we might as well open at get time // and then call update like we do in the else. // Can happen if the user sets the tez flag after the session was established. LOG.info("Tez session hasn't been created yet. Opening session"); session.open(nonConfResources); } else { LOG.info("Session is already open"); session.ensureLocalResources(conf, nonConfResources); } }
/** * Ensures that the Tez Session is open and the AM has all necessary jars configured. */ void updateSession(TezSessionState session, JobConf jobConf, Path scratchDir, String[] inputOutputJars, Map<String,LocalResource> extraResources) throws Exception { final boolean missingLocalResources = !session .hasResources(inputOutputJars); TezClient client = session.getSession(); // TODO null can also mean that this operation was interrupted. Should we really try to re-create the session in that case ? if (client == null) { // can happen if the user sets the tez flag after the session was // established LOG.info("Tez session hasn't been created yet. Opening session"); session.open(conf, inputOutputJars); } else { LOG.info("Session is already open"); // Ensure the open session has the necessary resources (StorageHandler) if (missingLocalResources) { LOG.info("Tez session missing resources," + " adding additional necessary resources"); client.addAppMasterLocalFiles(extraResources); } session.refreshLocalResourcesFromConf(conf); } }
@Test public void testCloseAndOpenDefault() throws Exception { poolManager = new TestTezSessionPoolManager(); TezSessionState session = Mockito.mock(TezSessionState.class); Mockito.when(session.isDefault()).thenReturn(false); Mockito.when(session.getConf()).thenReturn(conf); poolManager.reopen(session); Mockito.verify(session).close(false); Mockito.verify(session).open(Mockito.<TezSessionState.HiveResources>any()); }
startSs.tezSessionState.open(); } else { startSs.tezSessionState.beginOpen(null, console);
/** Reopens the session that was found to not be running. */ public void reopenSession(TezSessionState sessionState, HiveConf conf, String[] additionalFiles, boolean keepTmpDir) throws Exception { HiveConf sessionConf = sessionState.getConf(); if (sessionConf != null && sessionConf.get(TezConfiguration.TEZ_QUEUE_NAME) != null) { // user has explicitly specified queue name conf.set(TezConfiguration.TEZ_QUEUE_NAME, sessionConf.get(TezConfiguration.TEZ_QUEUE_NAME)); } else { // default queue name when the initial session was created if (sessionState.getQueueName() != null) { conf.set(TezConfiguration.TEZ_QUEUE_NAME, sessionState.getQueueName()); } } // TODO: close basically resets the object to a bunch of nulls. // We should ideally not reuse the object because it's pointless and error-prone. sessionState.close(keepTmpDir); // Clean up stuff. sessionState.open(conf, additionalFiles); }
startSs.tezSessionState.open(startSs.sessionConf); // should use conf on session start-up } else { startSs.tezSessionState.beginOpen(startSs.sessionConf, null, console);
Mockito.verify(session).open(Mockito.<TezSessionState.HiveResources>any());
public void open(HiveConf conf) throws IOException, LoginException, URISyntaxException, TezException { open(conf, null); }
public void startPool() throws Exception { this.inited = true; for (int i = 0; i < blockingQueueLength; i++) { HiveConf newConf = new HiveConf(initConf); TezSessionState sessionState = defaultQueuePool.take(); newConf.set("tez.queue.name", sessionState.getQueueName()); sessionState.open(newConf); openSessions.add(sessionState); defaultQueuePool.put(sessionState); } }
public void closeAndOpen(TezSessionState sessionState, HiveConf conf, String[] additionalFiles, boolean keepTmpDir) throws Exception { HiveConf sessionConf = sessionState.getConf(); if (sessionConf != null && sessionConf.get("tez.queue.name") != null) { conf.set("tez.queue.name", sessionConf.get("tez.queue.name")); } close(sessionState, keepTmpDir); sessionState.open(conf, additionalFiles); openSessions.add(sessionState); }
/** * @param conf HiveConf that is used to initialize the session * @param queueName could be null. Set in the tez session. * @param doOpen * @return * @throws Exception */ private TezSessionState getNewSessionState(HiveConf conf, String queueName, boolean doOpen) throws Exception { TezSessionState retTezSessionState = createSession(TezSessionState.makeSessionId()); if (queueName != null) { conf.set("tez.queue.name", queueName); } String what = "Created"; if (doOpen) { retTezSessionState.open(conf); openSessions.add(retTezSessionState); what = "Started"; } LOG.info(what + " a new session for queue: " + queueName + " session id: " + retTezSessionState.getSessionId()); return retTezSessionState; }
startSs.tezSessionState.open(startSs.conf); // should use conf on session start-up
/** * Ensures that the Tez Session is open and the AM has all necessary jars configured. */ void updateSession(TezSessionState session, JobConf jobConf, Path scratchDir, String[] inputOutputJars, Map<String,LocalResource> extraResources) throws Exception { final boolean missingLocalResources = !session .hasResources(inputOutputJars); if (!session.isOpen()) { // can happen if the user sets the tez flag after the session was // established LOG.info("Tez session hasn't been created yet. Opening session"); session.open(conf, inputOutputJars); } else { LOG.info("Session is already open"); // Ensure the open session has the necessary resources (StorageHandler) if (missingLocalResources) { LOG.info("Tez session missing resources," + " adding additional necessary resources"); session.getSession().addAppMasterLocalFiles(extraResources); } session.refreshLocalResourcesFromConf(conf); } }