@Override public void setDriver(LensDriver driver) { super.setDriver(driver); String[] allowedUsers = driver.getConf().getStrings(ALLOWED_USERS); String[] disallowedUsers = driver.getConf().getStrings(DISALLOWED_USERS); if (allowedUsers != null && disallowedUsers != null) { throw new IllegalStateException("You can't specify both allowed users and disallowed users"); } this.allowedUsers = allowedUsers == null ? null : Sets.newHashSet(allowedUsers); this.disallowedUsers = disallowedUsers == null ? null : Sets.newHashSet(disallowedUsers); }
@Override public void setDriver(LensDriver driver) { this.driver = driver; log.debug("The Driver for this driver query hook is {}", driver.getFullyQualifiedName()); }
driver.configure(LensServerConf.getConfForDrivers(), driverType, driverName); driver.registerDriverEventListener(driverEventListener); drivers.put(driver.getFullyQualifiedName(), driver); log.info("Driver {} for type {} is loaded", driverPath.getName(), driverType); } catch (Exception e) {
out.writeUTF(driverEntry.getKey()); out.writeUTF(driver.getClass().getName()); driver.writeExternal(out); out.writeBoolean(isDriverAvailable); if (isDriverAvailable) { out.writeUTF(ctx.getSelectedDriver().getFullyQualifiedName());
@Test public void testLoadingMultipleDrivers() { Collection<LensDriver> drivers = queryService.getDrivers(); assertEquals(drivers.size(), 4); Set<String> driverNames = new HashSet<>(drivers.size()); for (LensDriver driver : drivers) { assertEquals(driver.getConf().get("lens.driver.test.drivername"), driver.getFullyQualifiedName()); driverNames.add(driver.getFullyQualifiedName()); } assertTrue(driverNames.containsAll(Arrays.asList("hive/hive1", "hive/hive2", "jdbc/jdbc1", "mock/fail1"))); }
driverQueryContext.setQuery(driverQueries.get(driver)); try { driverQueryContext.setDriverQueryPlan(driver.explain(qctx)); succeededOnAtleastOneDriver = true; } catch (Exception e) { String expMsg = LensUtil.getCauseMessage(e); driverQueryContext.setDriverQueryPlanGenerationError(e); detailedFailureCause.append("\n Driver :").append(driver.getFullyQualifiedName()); detailedFailureCause.append(" Cause :" + expMsg); if (failureCause != null && !failureCause.equals(expMsg)) {
MethodMetricsContext rewriteGauge = MethodMetricsFactory.createMethodGauge(ctx.getDriverConf(driver), true, REWRITE_GAUGE); log.info("Calling preRewrite hook for driver {}", driver.getFullyQualifiedName()); driver.getQueryHook().preRewrite(ctx); cause = rewriterRunnable.getCause(); } else { log.info("Calling postRewrite hook for driver {}", driver.getFullyQualifiedName()); driver.getQueryHook().postRewrite(ctx); DRIVER_ESTIMATE_GAUGE); log.info("Calling preEstimate hook for driver {}", driver.getFullyQualifiedName()); driver.getQueryHook().preEstimate(ctx); estimateRunnable.run(); succeeded = estimateRunnable.isSucceeded(); log.error("Estimate failed for driver {} cause: {}", driver, failureCause); } else { log.info("Calling postRewrite hook for driver {}", driver.getFullyQualifiedName()); driver.getQueryHook().postEstimate(ctx);
&& ctx.getSelectedDriver() instanceof JDBCDriver)) && !getDriverRetryPolicy(ctx).hasExhaustedRetries(ctx)) { log.info("query {} will be retried on the same driver {}", ctx.getQueryHandle(), ctx.getSelectedDriver().getFullyQualifiedName()); ctx.extractFailedAttempt(); ctx.setStatus(QueryStatus.getQueuedStatus()); ctx.getSelectedDriver().closeQuery(ctx.getQueryHandle()); return queuedQueries.add(ctx); } else if (!getServerRetryPolicy(ctx).hasExhaustedRetries(ctx)) { } catch (LensException e) { log.error("driver {} gave up on query {} and it will not be retried on any other driver since rewrite failed", selectedDriver.getFullyQualifiedName(), e); ctx.setStatus(new QueryStatus(1.0f, null, FAILED, ctx.getStatus().getStatusMessage(), false, null, ctx.getStatus().getErrorMessage(), ctx.getStatus().getLensErrorTO())); return false; log.info("driver {} gave up on query {} and it will be retried on {}", selectedDriver.getFullyQualifiedName(), ctx.getQueryHandle(), ctx.getSelectedDriver().getFullyQualifiedName()); ctx.extractFailedAttempt(selectedDriver); ctx.setStatus(QueryStatus.getQueuedStatus()); selectedDriver.closeQuery(ctx.getQueryHandle()); return queuedQueries.add(ctx);
driver = driverCls.newInstance(); String[] driverTypeAndName = StringUtils.split(driverQualifiedName, '/'); driver.configure(conf, driverTypeAndName[0], driverTypeAndName[1]); } catch (Exception e) { log.error("Could not instantiate driver:{} represented by class {}", driverQualifiedName, driverClsName, e); driver.readExternal(in);
/** * Gets the driver resultset. * * @param queryHandle the query handle * @return the driver resultset * @throws LensException the lens exception */ LensResultSet getDriverResultset(QueryHandle queryHandle) throws LensException { return allQueries.get(queryHandle).getSelectedDriver().fetchResultSet(allQueries.get(queryHandle)); }
&& !queryCtx.getStatus().executed() && !queryCtx.getStatus().finished()) { log.info("Registering for query {} completion notification", ctx.getQueryHandleString()); queryCtx.getSelectedDriver().registerForCompletionNotification(ctx, totalWaitTime, listener); try { queryCtx = getUpdatedQueryContext(sessionHandle, handle, true); // If the query is already purged queryCtx = null if (queryCtx != null && queryCtx.getStatus().isResultSetAvailable()) { resultSet = queryCtx.getSelectedDriver().fetchResultSet(queryCtx); if (resultSet instanceof PartiallyFetchedInMemoryResultSet) { PartiallyFetchedInMemoryResultSet partialnMemoryResult = (PartiallyFetchedInMemoryResultSet) resultSet;
@Override public QueryPlan explain(final String requestId, LensSessionHandle sessionHandle, String query, LensConf lensConf) throws LensException { try { log.info("Explain: session:{} query:{}", sessionHandle, query); acquire(sessionHandle); Configuration qconf = getLensConf(sessionHandle, lensConf); ExplainQueryContext explainQueryContext = new ExplainQueryContext(requestId, query, getSession(sessionHandle) .getLoggedInUser(), lensConf, qconf, drivers.values()); explainQueryContext.setLensSessionIdentifier(sessionHandle.getPublicId().toString()); accept(query, qconf, SubmitOp.EXPLAIN); rewriteAndSelect(explainQueryContext); addSessionResourcesToDriver(explainQueryContext); return explainQueryContext.getSelectedDriver().explain(explainQueryContext).toQueryPlan(); } catch (UnsupportedEncodingException e) { throw new LensException(e); } finally { release(sessionHandle); } }
/** * Creates the query context * @param conf the conf * @param queryName the name of query * @return the query context */ protected QueryContext createContext(Configuration conf, String queryName) { final LensDriver mockDriver = new MockDriver(); try { mockDriver.configure(conf, null, null); } catch (LensException e) { Assert.fail(e.getMessage()); } QueryContext ctx = QueryContext.createContextWithSingleDriver("test writer query", "testuser", new LensConf(), conf, mockDriver, null, false); ctx.setSelectedDriver(mockDriver); ctx.setQueryName(queryName); return ctx; }
private boolean cancelQuery(@NonNull QueryHandle queryHandle) throws LensException { QueryContext ctx = allQueries.get(queryHandle); if (ctx == null) { log.info("Could not cancel query {} as it has been purged already", queryHandle); return false; } synchronized (ctx) { updateStatus(queryHandle); if (ctx.finished()) { log.info("Could not cancel query {} as it has finished execution already", queryHandle); return false; } if (ctx.isLaunching()) { boolean launchCancelled = ctx.getQueryLauncher().cancel(true); log.info("query launch cancellation success : {}", launchCancelled); } if (ctx.launched() || ctx.running()) { if (!ctx.getSelectedDriver().cancelQuery(queryHandle)) { log.info("Could not cancel query {}", queryHandle); return false; } } log.info("Query {} cancelled successfully", queryHandle); setCancelledStatus(ctx, "Query is cancelled"); return true; } }
driverQueryContext.setQuery(driverQueries.get(driver)); try { driverQueryContext.setDriverQueryPlan(driver.explain(qctx)); succeededOnAtleastOneDriver = true; } catch (Exception e) { String expMsg = LensUtil.getCauseMessage(e); driverQueryContext.setDriverQueryPlanGenerationError(e); detailedFailureCause.append("\n Driver :").append(driver.getFullyQualifiedName()); detailedFailureCause.append(" Cause :" + expMsg); if (failureCause != null && !failureCause.equals(expMsg)) {
/** * Instantiates a new finished query. * * @param ctx the ctx */ FinishedQuery(QueryContext ctx) { this.ctx = ctx; if (ctx.getEndTime() == 0) { this.finishTime = new Date(); ctx.setEndTime(this.finishTime.getTime()); } else { this.finishTime = new Date(ctx.getEndTime()); } if (ctx.isResultAvailableInDriver()) { try { driverRS = ctx.getSelectedDriver().fetchResultSet(getCtx()); } catch (Exception e) { log.error( "Error while getting result set form driver {}. Driver result set based purging logic will be ignored", ctx.getSelectedDriver(), e); } } } public boolean canBePurged() {
@Override public void setDriver(LensDriver driver) { this.driver = driver; log.debug("The Driver for this driver query hook is {}", driver.getFullyQualifiedName()); }
@Override public void setDriver(LensDriver driver) { super.setDriver(driver); String[] allowedUsers = driver.getConf().getStrings(ALLOWED_USERS); String[] disallowedUsers = driver.getConf().getStrings(DISALLOWED_USERS); if (allowedUsers != null && disallowedUsers != null) { throw new IllegalStateException("You can't specify both allowed users and disallowed users"); } this.allowedUsers = allowedUsers == null ? null : Sets.newHashSet(allowedUsers); this.disallowedUsers = disallowedUsers == null ? null : Sets.newHashSet(disallowedUsers); }
} else { QueryContext context = queryService.getQueryContext(handle); long creationTime = ((InMemoryResultSet) (context.getSelectedDriver().fetchResultSet(context))).getCreationTime(); long inMemoryResultsetTTLSecs = LensServerConf.getHiveConf().getInt(LensConfConstants.INMEMORY_RESULT_SET_TTL_SECS,
private LensDriver getDriverFromName(Collection<LensDriver> drivers) { for (LensDriver driver : drivers) { if (driverName.equals(driver.getFullyQualifiedName())) { return driver; } } return null; }