@Override public void setDriver(LensDriver driver) { super.setDriver(driver); String[] allowedUsers = driver.getConf().getStrings(ALLOWED_USERS); String[] disallowedUsers = driver.getConf().getStrings(DISALLOWED_USERS); if (allowedUsers != null && disallowedUsers != null) { throw new IllegalStateException("You can't specify both allowed users and disallowed users"); } this.allowedUsers = allowedUsers == null ? null : Sets.newHashSet(allowedUsers); this.disallowedUsers = disallowedUsers == null ? null : Sets.newHashSet(disallowedUsers); }
@Override public void setDriver(LensDriver driver) { super.setDriver(driver); String[] allowedUsers = driver.getConf().getStrings(ALLOWED_USERS); String[] disallowedUsers = driver.getConf().getStrings(DISALLOWED_USERS); if (allowedUsers != null && disallowedUsers != null) { throw new IllegalStateException("You can't specify both allowed users and disallowed users"); } this.allowedUsers = allowedUsers == null ? null : Sets.newHashSet(allowedUsers); this.disallowedUsers = disallowedUsers == null ? null : Sets.newHashSet(disallowedUsers); }
@Override public void init(LensDriver lensDriver) { queryCostTypeDecider = new RangeBasedQueryCostTypeDecider( lensDriver.getConf().get(DRIVER_COST_TYPE_RANGES, DRIVER_QUERY_COST_TYPE_DEFAULT_RANGES)); }
@Override public void setDriver(LensDriver driver) { super.setDriver(driver); Class<? extends Parser<T>> parserClass = (Class<? extends Parser<T>>) driver.getConf().getClass(QUERY_COST_PARSER, DEFAULT_QUERY_COST_PARSER, Parser.class); try { this.parser = parserClass.newInstance(); } catch (InstantiationException | IllegalAccessException e) { throw new IllegalArgumentException("Couldn't initialize query cost parser from class " + parserClass); } allowedCosts = parseAllowedRangeSet(driver.getConf().get(ALLOWED_RANGE_SETS)); allowedCosts.removeAll(parseDisallowedRangeSet(driver.getConf().get(DISALLOWED_RANGE_SETS))); }
@Override public void setDriver(LensDriver driver) { super.setDriver(driver); Class<? extends Parser<T>> parserClass = (Class<? extends Parser<T>>) driver.getConf().getClass(QUERY_COST_PARSER, DEFAULT_QUERY_COST_PARSER, Parser.class); try { this.parser = parserClass.newInstance(); } catch (InstantiationException | IllegalAccessException e) { throw new IllegalArgumentException("Couldn't initialize query cost parser from class " + parserClass); } allowedCosts = parseAllowedRangeSet(driver.getConf().get(ALLOWED_RANGE_SETS)); allowedCosts.removeAll(parseDisallowedRangeSet(driver.getConf().get(DISALLOWED_RANGE_SETS))); }
/** * Gets the driver query conf. * * @param driver the driver * @param queryConf the query conf * @return the final query conf */ private Configuration mergeConf(LensDriver driver, Configuration queryConf) { Configuration conf = new Configuration(driver.getConf()); for (Map.Entry<String, String> entry : queryConf) { if (!conf.getFinalParameters().contains(entry.getKey())) { conf.set(entry.getKey(), entry.getValue()); } } conf.setClassLoader(queryConf.getClassLoader()); return conf; }
/** * Gets the driver query conf. * * @param driver the driver * @param queryConf the query conf * @return the final query conf */ private Configuration mergeConf(LensDriver driver, Configuration queryConf) { Configuration conf = new Configuration(driver.getConf()); for (Map.Entry<String, String> entry : queryConf) { if (!conf.getFinalParameters().contains(entry.getKey())) { conf.set(entry.getKey(), entry.getValue()); } } conf.setClassLoader(queryConf.getClassLoader()); return conf; }
@Override public void init(LensDriver lensDriver) throws LensException { QueryCostTypeDecider queryCostTypeDecider = new RangeBasedQueryCostTypeDecider( lensDriver.getConf().get(DRIVER_COST_TYPE_RANGES, DRIVER_QUERY_COST_TYPE_DEFAULT_RANGES)); this.queryCost = new StaticQueryCost(lensDriver.getConf().getDouble(DRIVER_QUERY_COST, DEFAULT_DRIVER_QUERY_COST)); this.queryCost.setQueryCostType(queryCostTypeDecider.decideCostType(this.queryCost)); }
@BeforeMethod public void setUp() throws Exception { driver = mock(LensDriver.class); conf = new Configuration(); when(driver.getConf()).thenReturn(conf); ctx = mock(AbstractQueryContext.class); }
@BeforeMethod public void setUp() throws Exception { driver = mock(LensDriver.class); Configuration conf = new Configuration(); conf.set(QueryCostBasedQueryHook.ALLOWED_RANGE_SETS, "[10, 100]"); conf.set(QueryCostBasedQueryHook.DISALLOWED_RANGE_SETS, "[40, 50]"); when(driver.getConf()).thenReturn(conf); ctx = mock(AbstractQueryContext.class); hook.setDriver(driver); }
for (LensDriver driver : drivers) { log.debug("Cost on driver {}: {}", driver, ctx.getDriverQueryCost(driver)); log.debug("Driver ratio: {}", driver.getConf().getDouble(DRIVER_WEIGHT, DEFAULT_DRIVER_WEIGHT)); driverRatioSum = driverRatioSum + driver.getConf().getDouble(DRIVER_WEIGHT, DEFAULT_DRIVER_WEIGHT); driverRatioSum -= driver.getConf().getDouble(DRIVER_WEIGHT, DEFAULT_DRIVER_WEIGHT); if (driverRatioSum <= 0){ return driver;
@BeforeMethod public void setUp() throws Exception { driver = mock(LensDriver.class); Configuration conf = new Configuration(); when(driver.getConf()).thenReturn(conf); ctx = mock(AbstractQueryContext.class); conf.set(UserBasedQueryHook.ALLOWED_USERS, "user1,user2"); conf.set(QueryCostBasedQueryHook.ALLOWED_RANGE_SETS, "[0, 10)"); conf.set("chain", UserBasedQueryHook.class.getName() + "," + QueryCostBasedQueryHook.class.getName()); hook = ChainedDriverQueryHook.from(conf, "chain"); hook.setDriver(driver); }
for (LensDriver driver : drivers) { log.debug("Cost on driver {}: {}", driver, ctx.getDriverQueryCost(driver)); log.debug("Driver ratio: {}", driver.getConf().getDouble(DRIVER_WEIGHT, DEFAULT_DRIVER_WEIGHT)); driverRatioSum = driverRatioSum + driver.getConf().getDouble(DRIVER_WEIGHT, DEFAULT_DRIVER_WEIGHT); driverRatioSum -= driver.getConf().getDouble(DRIVER_WEIGHT, DEFAULT_DRIVER_WEIGHT); if (driverRatioSum <= 0){ return driver;
@Test public void testLoadingMultipleDrivers() { Collection<LensDriver> drivers = queryService.getDrivers(); assertEquals(drivers.size(), 4); Set<String> driverNames = new HashSet<>(drivers.size()); for (LensDriver driver : drivers) { assertEquals(driver.getConf().get("lens.driver.test.drivername"), driver.getFullyQualifiedName()); driverNames.add(driver.getFullyQualifiedName()); } assertTrue(driverNames.containsAll(Arrays.asList("hive/hive1", "hive/hive2", "jdbc/jdbc1", "mock/fail1"))); }
/** * Normalized cost of an update period. update period weight multiplied by number of partitions of that update period. * * @param updatePeriod * @param driver * @return normalized cost. * @throws LensException */ private double getNormalizedUpdatePeriodCost(final UpdatePeriod updatePeriod, LensDriver driver) throws LensException { double weight = driver.getConf().getDouble(UPDATE_PERIOD_WEIGHT_PREFIX + updatePeriod.name().toLowerCase(), updatePeriod.getNormalizationFactor()); return weight * updatePeriod.weight() / UpdatePeriod.DAILY.weight(); }
/** * Normalized cost of an update period. update period weight multiplied by number of partitions of that update period. * * @param updatePeriod * @param driver * @return normalized cost. * @throws LensException */ private double getNormalizedUpdatePeriodCost(final UpdatePeriod updatePeriod, LensDriver driver) throws LensException { double weight = driver.getConf().getDouble(UPDATE_PERIOD_WEIGHT_PREFIX + updatePeriod.name().toLowerCase(), updatePeriod.getNormalizationFactor()); return weight * updatePeriod.weight() / UpdatePeriod.DAILY.weight(); }
@BeforeTest public void setUp() { driver = mock(LensDriver.class); when(driver.getConf()).thenReturn(new Configuration()); queryContext = mock(AbstractQueryContext.class); calculator.init(driver); ImmutableMap<String, Double> tableWeights = new ImmutableMap.Builder<String, Double>().build(); FactPartition fp1 = mockFactPartition(DAILY, tableWeights, 0.7); FactPartition fp2 = mockFactPartition(HOURLY, tableWeights, 0.8); FactPartition fp3 = mockFactPartition(SECONDLY, tableWeights, 0.4); FactPartition fp4 = mockFactPartition(MONTHLY, tableWeights, 0); when(queryContext.getTableWeights(driver)).thenReturn(tableWeights); HashMap<String, Set<?>> partitions = new HashMap<>(); partitions.put("st1", Sets.newHashSet(fp1, fp2)); partitions.put("st2", Sets.newHashSet(fp3, fp4)); partitions.put("st3", Sets.newHashSet(latest)); DriverQueryPlan plan = mock(DriverQueryPlan.class); when(queryContext.getDriverRewriterPlan(driver)).thenReturn(plan); when(plan.getPartitions()).thenReturn(partitions); when(calculator.getAllPartitions(queryContext, driver)).thenReturn(partitions); }