/** * Get {@link CubeDimensionTable} object corresponding to the name * * @param tableName The cube dimension name * @return Returns CubeDimensionTable if table name passed is a dimension table * @throws LensException if there is no dimension table with the name */ public CubeDimensionTable getDimensionTable(String tableName) throws LensException { return getDimensionTable(tableName, true); } private CubeDimensionTable getDimensionTable(String tableName, boolean throwException)
/** * Get {@link CubeDimensionTable} object corresponding to the name * * @param tableName The cube dimension name * @return Returns CubeDimensionTable if table name passed is a dimension table * @throws LensException if there is no dimension table with the name */ public CubeDimensionTable getDimensionTable(String tableName) throws LensException { return getDimensionTable(tableName, true); }
private void updateDimCache(String dimTblName) throws LensException { if (enableCaching) { allDimTables.put(dimTblName.trim().toLowerCase(), getDimensionTable(refreshTable(dimTblName))); } }
private void updateDimCache(String dimTblName) throws LensException { if (enableCaching) { allDimTables.put(dimTblName.trim().toLowerCase(), getDimensionTable(refreshTable(dimTblName))); } } public void alterCubeDimensionTable(XDimensionTable dimensionTable) throws LensException, HiveException {
public XDimensionTable getXDimensionTable(String dimTable) throws LensException { return getXDimensionTable(getDimensionTable(dimTable)); } public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException {
public XDimensionTable getXDimensionTable(String dimTable) throws LensException { return getXDimensionTable(getDimensionTable(dimTable)); }
private CubeDimensionTable checkDimTableStorage(LensSessionHandle sessionid, String dimTable, String storage) throws HiveException, LensException { CubeMetastoreClient client = getClient(sessionid); CubeDimensionTable cdt = client.getDimensionTable(dimTable); client.verifyStorageExists(cdt, storage); return cdt; }
@Override public List<String> getDimTableStorages(LensSessionHandle sessionid, String dimension) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeDimensionTable dimTable = getClient(sessionid).getDimensionTable(dimension); return new ArrayList<>(dimTable.getStorages()); } }
boolean dimPartitionExists(String dimTblName, String storageName, Map<String, Date> partitionTimestamps) throws HiveException, LensException { String storageTableName = getFactOrDimtableStorageTableName(dimTblName, storageName); return partitionExists(storageTableName, getDimensionTable(dimTblName).getSnapshotDumpPeriods().get(storageName), partitionTimestamps); }
boolean dimPartitionExists(String dimTblName, String storageName, Map<String, Date> partitionTimestamps) throws HiveException, LensException { String storageTableName = getFactOrDimtableStorageTableName(dimTblName, storageName); return partitionExists(storageTableName, getDimensionTable(dimTblName).getSnapshotDumpPeriods().get(storageName), partitionTimestamps); }
/** * Create a cube dimension table * * @param dimName The dimension name to which the dim-table belongs to * @param dimTblName dimension table name * @param columns Columns of the dimension table * @param weight Weight of the dimension table * @param dumpPeriods Storage names and their dump periods on which dimension is available * @param properties properties of dimension table * @param storageTableDescs Map of storage to its storage table description * @throws LensException */ public void createCubeDimensionTable(String dimName, String dimTblName, List<FieldSchema> columns, double weight, Map<String, UpdatePeriod> dumpPeriods, Map<String, String> properties, Map<String, StorageTableDesc> storageTableDescs) throws LensException { CubeDimensionTable dimTable = new CubeDimensionTable(dimName, dimTblName, columns, weight, dumpPeriods, properties); createCubeTable(dimTable, storageTableDescs); // do a get to update cache getDimensionTable(dimTblName); }
/** * Create a cube dimension table * * @param dimName The dimension name to which the dim-table belongs to * @param dimTblName dimension table name * @param columns Columns of the dimension table * @param weight Weight of the dimension table * @param storageNames Storages on which dimension is available without any dumps * @param properties Properties of dimension table * @param storageTableDescs Map of storage to its storage table description * @throws LensException */ public void createCubeDimensionTable(String dimName, String dimTblName, List<FieldSchema> columns, double weight, Set<String> storageNames, Map<String, String> properties, Map<String, StorageTableDesc> storageTableDescs) throws LensException { CubeDimensionTable dimTable = new CubeDimensionTable(dimName, dimTblName, columns, weight, storageNames, properties); createCubeTable(dimTable, storageTableDescs); // do a get to update cache getDimensionTable(dimTblName); }
/** * Drop the dimension table * * @param dimTblName dim table name * @param cascade If true, will drop all the dimension storages * @throws HiveException */ public void dropDimensionTable(String dimTblName, boolean cascade) throws LensException { getTableWithTypeFailFast(dimTblName, CubeTableType.DIM_TABLE); CubeDimensionTable dim = getDimensionTable(dimTblName); if (cascade) { for (String storage : dim.getStorages()) { dropStorageFromDim(dimTblName, storage, false); } } dropHiveTable(dimTblName); allDimTables.remove(dimTblName.trim().toLowerCase()); }
private void dropStorageFromDim(String dimTblName, String storage, boolean updateDimTbl) throws LensException { CubeDimensionTable cdt = getDimensionTable(dimTblName); String storageTableName = getFactOrDimtableStorageTableName(dimTblName, storage); dropHiveTable(storageTableName); latestLookupCache.remove(storageTableName.trim().toLowerCase()); if (updateDimTbl) { cdt.dropStorage(storage); alterCubeTable(dimTblName, getTableWithTypeFailFast(dimTblName, CubeTableType.DIM_TABLE), cdt); updateDimCache(dimTblName); } }
@Override public void dropAllStoragesOfDimTable(LensSessionHandle sessionid, String dimTblName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); CubeDimensionTable tab = msClient.getDimensionTable(dimTblName); int total = tab.getStorages().size(); int i = 0; List<String> storageNames = new ArrayList<>(tab.getStorages()); for (String s : storageNames) { msClient.dropStorageFromDim(dimTblName, s); log.info("Dropped storage " + s + " from dimension table " + dimTblName + " [" + ++i + "/" + total + "]"); } log.info("Dropped " + total + " storages from dimension table " + dimTblName); } catch (HiveException exc) { throw new LensException(exc); } }
private void dropStorageFromDim(String dimTblName, String storage, boolean updateDimTbl) throws LensException { checkIfAuthorized(); CubeDimensionTable cdt = getDimensionTable(dimTblName); String storageTableName = getFactOrDimtableStorageTableName(dimTblName, storage); dropHiveTable(storageTableName); latestLookupCache.remove(storageTableName.trim().toLowerCase()); if (updateDimTbl) { cdt.dropStorage(storage); alterCubeTable(dimTblName, getTableWithTypeFailFast(dimTblName, CubeTableType.DIM_TABLE), cdt); updateDimCache(dimTblName); } }
@Override public void dropStorageOfDimTable(LensSessionHandle sessionid, String dimTblName, String storage) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); CubeDimensionTable tab = msClient.getDimensionTable(dimTblName); if (!tab.getStorages().contains(storage)) { throw new NotFoundException("Storage " + storage + " not found for dimension " + dimTblName); } msClient.dropStorageFromDim(dimTblName, storage); log.info("Dropped storage " + storage + " from dimension table " + dimTblName); } catch (HiveException exc) { throw new LensException(exc); } }
public XStorageTableElement getStorageOfDim(LensSessionHandle sessionid, String dimTblName, String storageName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); CubeDimensionTable dimTable = msClient.getDimensionTable(dimTblName); XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } return tblElement; } }
@Override public void addDimTableStorage(LensSessionHandle sessionid, String dimTblName, XStorageTableElement storageTable) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); CubeDimensionTable dimTable = msClient.getDimensionTable(dimTblName); UpdatePeriod period = null; if (storageTable.getUpdatePeriods() != null && !storageTable.getUpdatePeriods().getUpdatePeriod().isEmpty()) { period = UpdatePeriod.valueOf(storageTable.getUpdatePeriods().getUpdatePeriod().get(0).name()); } msClient.addStorage(dimTable, storageTable.getStorageName(), period, JAXBUtils.storageTableDescFromXStorageTableDesc(storageTable.getTableDesc())); log.info("Added storage " + storageTable.getStorageName() + " for dimension table " + dimTblName + " with update period " + period); } }
@Test public void testExpressionPruningForInvalidDim() throws Exception { Configuration newConf = new Configuration(conf); newConf.set(CubeQueryConfUtil.DRIVER_SUPPORTED_STORAGES, "C2"); // cityaddress = //new ExprSpec("concat(citydim.name, \":\", statedim.name, \":\", countrydim.name, \":\", zipdim.code)", null, // null), new ExprSpec("concat(citydim.name, \":\", statedim.name)", null, null))); // since zipdim is not available in storage C2, first expression should be have been pruned // And joining with statedim for second expression is not possible because of stateid missing in C2 tables // or citydim.name missing in c2 tables. CubeQueryContext ctx = rewriteCtx("select citydim.name, cityaddress from citydim", newConf); Assert.assertEquals(ctx.getDimPruningMsgs().get(ctx.getMetastoreClient().getDimension("citydim")) .get(ctx.getMetastoreClient().getDimensionTable("citytable")).size(), 1); CandidateTablePruneCause pruningMsg = ctx.getDimPruningMsgs().get(ctx.getMetastoreClient().getDimension("citydim")) .get(ctx.getMetastoreClient().getDimensionTable("citytable")).get(0); Assert.assertEquals(pruningMsg.getCause(), CandidateTablePruneCode.EXPRESSION_NOT_EVALUABLE); Assert.assertTrue(pruningMsg.getMissingExpressions().contains("cityaddress")); }