/** * Get {@link Cube} object corresponding to the name * * @param tableName The cube name * @return Returns Dimension if table name passed is a Dimension * @throws LensException if the table name passed is not a Dimension */ public Dimension getDimension(String tableName) throws LensException { return getDimension(tableName, true); } private Dimension getDimension(String tableName, boolean throwException) throws LensException {
@Test public void testExpressionPruningForInvalidDim() throws Exception { Configuration newConf = new Configuration(conf); newConf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); // cityaddress = //new ExprSpec("concat(citydim.name, \":\", statedim.name, \":\", countrydim.name, \":\", zipdim.code)", null, // null), new ExprSpec("concat(citydim.name, \":\", statedim.name)", null, null))); // since zipdim is not available in storage C2, first expression should be have been pruned // And joining with statedim for second expression is not possible because of stateid missing in C2 tables // or citydim.name missing in c2 tables. CubeQueryContext ctx = rewriteCtx("select citydim.name, cityaddress from citydim", newConf); Assert.assertEquals(ctx.getDimPruningMsgs().get(ctx.getMetastoreClient().getDimension("citydim")) .get(ctx.getMetastoreClient().getDimensionTable("citytable")).size(), 1); CandidateTablePruneCause pruningMsg = ctx.getDimPruningMsgs().get(ctx.getMetastoreClient().getDimension("citydim")) .get(ctx.getMetastoreClient().getDimensionTable("citytable")).get(0); Assert.assertEquals(pruningMsg.getCause(), CandidateTablePruneCode.EXPRESSION_NOT_EVALUABLE); Assert.assertTrue(pruningMsg.getMissingExpressions().contains("cityaddress")); }
/** * Get {@link Cube} object corresponding to the name * * @param tableName The cube name * @return Returns Dimension if table name passed is a Dimension * @throws LensException if the table name passed is not a Dimension */ public Dimension getDimension(String tableName) throws LensException { return getDimension(tableName, true); }
private Dimension getDimension(String tableName, boolean throwException) throws LensException { if (tableName == null) { return null; } tableName = tableName.trim().toLowerCase(); Dimension dim = allDims.get(tableName); if (dim == null) { synchronized (allDims) { if (!allDims.containsKey(tableName)) { Table tbl = getTableWithType(tableName, CubeTableType.DIMENSION, throwException); dim = tbl == null ? null : getDimension(tbl); if (enableCaching && dim != null) { allDims.put(tableName, dim); } } else { dim = allDims.get(tableName); } } } return dim; }
private Dimension getDimension(String tableName, boolean throwException) throws LensException { if (tableName == null) { return null; } tableName = tableName.trim().toLowerCase(); Dimension dim = allDims.get(tableName); if (dim == null) { synchronized (allDims) { if (!allDims.containsKey(tableName)) { Table tbl = getTableWithType(tableName, CubeTableType.DIMENSION, throwException); dim = tbl == null ? null : getDimension(tbl); if (enableCaching && dim != null) { allDims.put(tableName, dim); } } else { dim = allDims.get(tableName); } } } return dim; }
public void alterDimension(String dimName, Dimension newDim) throws HiveException, LensException { Table tbl = getTableWithTypeFailFast(dimName, CubeTableType.DIMENSION); alterCubeTable(dimName, tbl, newDim); if (enableCaching) { allDims.put(dimName.trim().toLowerCase(), getDimension(refreshTable(dimName))); } }
/** * Create dimension in metastore defined by {@link Dimension} object * * @param dim the {@link Dimension} object. * @throws LensException */ public void createDimension(Dimension dim) throws LensException { createCubeHiveTable(dim); // do a get to update cache getDimension(dim.getName()); }
public void alterDimension(String dimName, Dimension newDim) throws HiveException, LensException { checkIfAuthorized(); Table tbl = getTableWithTypeFailFast(dimName, CubeTableType.DIMENSION); alterCubeTable(dimName, tbl, newDim); if (enableCaching) { allDims.put(dimName.trim().toLowerCase(), getDimension(refreshTable(dimName))); } }
private void processJoinChains(CubeQueryContext cubeql) throws HiveException, LensException { for (JoinChain chain : cubeql.getJoinchains().values()) { Set<String> dims = chain.getIntermediateDimensions(); dims.add(chain.getDestTable()); for (String dim : dims) { Dimension dimension = cubeql.getMetastoreClient().getDimension(dim); dimensionToJoinChainsMap.computeIfAbsent(dimension, k -> new ArrayList<>()).add(chain); } } }
private void processJoinChains(CubeQueryContext cubeql) throws HiveException, LensException { for (JoinChain chain : cubeql.getJoinchains().values()) { Set<String> dims = chain.getIntermediateDimensions(); dims.add(chain.getDestTable()); for (String dim : dims) { Dimension dimension = cubeql.getMetastoreClient().getDimension(dim); dimensionToJoinChainsMap.computeIfAbsent(dimension, k -> new ArrayList<>()).add(chain); } } }
TableRelationship toDimToDimRelationship(CubeMetastoreClient client) throws HiveException, LensException { if (relationShip == null) { relationShip = new TableRelationship(from.getDestColumn(), client.getDimension(from.getDestTable()), to.getDestColumn(), client.getDimension(to.getDestTable()), to.isMapsToMany()); } return relationShip; }
@Override public XDimension getDimension(LensSessionHandle sessionid, String dimName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ return JAXBUtils.xdimensionFromDimension(getClient(sessionid).getDimension(dimName)); } }
TableRelationship toDimToDimRelationship(CubeMetastoreClient client) throws HiveException, LensException { if (relationShip == null) { relationShip = new TableRelationship(from.getDestColumn(), client.getDimension(from.getDestTable()), to.getDestColumn(), client.getDimension(to.getDestTable()), to.isMapsToMany()); } return relationShip; }
/** * Create dimension in metastore defined by {@link Dimension} object * * @param dim the {@link Dimension} object. * @throws LensException */ public void createDimension(Dimension dim) throws LensException { checkIfAuthorized(); createCubeHiveTable(dim); // do a get to update cache getDimension(dim.getName()); }
@Override public List<String> getAllDimTableNames(LensSessionHandle sessionid, String dimensionName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient client = getClient(sessionid); Dimension dimension = client.getDimension(dimensionName); if (dimensionName != null && dimension == null) { throw new LensException("Could not get table: " + dimensionName + " as a dimension"); } Collection<CubeDimensionTable> dims = client.getAllDimensionTables(dimension); List<String> dimNames = new ArrayList<>(dims.size()); for (CubeDimensionTable cdt : dims) { dimNames.add(cdt.getName()); } return dimNames; } }
@Test public void testDimensionQueryWithTwoRefCols() throws Exception { Configuration tConf = new Configuration(conf); tConf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, ""); CubeQueryContext cubeql = rewriteCtx("select citydim.zipcode, citydim.statename from" + " citydim", tConf); Set<String> candidateDims = new HashSet<>(); for (CandidateDim cdim : cubeql.getCandidateDims().get(cubeql.getMetastoreClient().getDimension("citydim"))) { candidateDims.add(cdim.getStorageTable()); } // city_table2 contains stateid, but not zipcode - it should have been removed. Assert.assertFalse(candidateDims.contains("city_table2")); }
@Test public void testChainsWithMultipleStorage() throws ParseException, HiveException, LensException { Configuration conf = new Configuration(hconf); conf.unset(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES); // supports all storages String dimOnlyQuery = "select testDim2.name, testDim2.cityStateCapital FROM testDim2 where " + TWO_DAYS_RANGE; CubeQueryRewriter driver = new CubeQueryRewriter(conf, hconf); CubeQueryContext rewrittenQuery = driver.rewrite(dimOnlyQuery); rewrittenQuery.toHQL(); Dimension citydim = CubeMetastoreClient.getInstance(hconf).getDimension("citydim"); Set<String> cdimTables = new HashSet<>(); for (CandidateDim cdim : rewrittenQuery.getCandidateDims().get(citydim)) { cdimTables.add(cdim.getName()); } Assert.assertTrue(cdimTables.contains("citytable")); Assert.assertTrue(cdimTables.contains("citytable2")); Assert.assertFalse(cdimTables.contains("citytable3")); Assert.assertFalse(cdimTables.contains("citytable4")); }
private void addAllChainedColsToFlattenedList(CubeMetastoreClient client, ObjectFactory objectFactory, AbstractBaseTable baseTbl, List<XFlattenedColumn> columnList) throws HiveException, LensException { for (JoinChain chain : baseTbl.getJoinChains()) { Dimension dim = client.getDimension(chain.getDestTable()); addAllDirectAttributesToFlattenedListFromDimension(objectFactory, dim, columnList, chain.getName()); addAllDirectExpressionsToFlattenedList(objectFactory, dim, columnList, chain.getName()); } }
/** * return Cube or Dimension relationship depending on the source table of the join chain. * * @param client * @return * @throws HiveException */ TableRelationship toCubeOrDimRelationship(CubeMetastoreClient client) throws HiveException, LensException { if (relationShip == null) { AbstractCubeTable fromTable = null; if (client.isCube(from.getDestTable())) { fromTable = (AbstractCubeTable) client.getCube(from.getDestTable()); } else if (client.isDimension(from.getDestTable())) { fromTable = client.getDimension(from.getDestTable()); } if (fromTable != null) { relationShip = new TableRelationship(from.getDestColumn(), fromTable, to.getDestColumn(), client.getDimension(to.getDestTable()), to.isMapsToMany()); } } return relationShip; }
private void validateDim(Dimension udim, Set<CubeDimAttribute> attrs, String basedim, String referdim) throws HiveException, LensException { assertTrue(client.tableExists(udim.getName())); Table dimTbl = client.getHiveTable(udim.getName()); assertTrue(client.isDimension(dimTbl)); Dimension dim = new Dimension(dimTbl); assertTrue(udim.equals(dim), "Equals failed for " + dim.getName()); assertTrue(udim.equals(client.getDimension(udim.getName()))); assertEquals(dim.getAttributes().size(), attrs.size()); assertNotNull(dim.getAttributeByName(basedim)); assertTrue(dim.getAttributeByName(basedim) instanceof BaseDimAttribute); if (referdim != null) { assertNotNull(dim.getAttributeByName(referdim)); assertTrue(dim.getAttributeByName(referdim) instanceof ReferencedDimAttribute); } assertEquals(udim.getAttributeNames().size() + udim.getExpressionNames().size(), dim.getAllFieldNames().size()); }