private void dropHiveTablesForStorage(String factName, String storage) throws LensException { CubeFactTable cft = getCubeFactTable(factName); Set<String> droppedTables = new HashSet<>(); for (Map.Entry updatePeriodEntry : cft.getStoragePrefixUpdatePeriodMap().get(storage).entrySet()) { UpdatePeriod updatePeriod = (UpdatePeriod) updatePeriodEntry.getKey(); String storageTableName = getStorageTableName(factName, storage, updatePeriod); if (!droppedTables.contains(storageTableName)) { dropHiveTable(storageTableName); } droppedTables.add(storageTableName); } }
private void dropHiveTablesForStorage(String factName, String storage) throws LensException{ CubeFactTable cft = getCubeFactTable(factName); Set<String> droppedTables = new HashSet<>(); for (Map.Entry updatePeriodEntry : cft.getStoragePrefixUpdatePeriodMap().get(storage).entrySet()) { UpdatePeriod updatePeriod = (UpdatePeriod) updatePeriodEntry.getKey(); String storageTableName = getStorageTableName(factName, storage, updatePeriod); if (!droppedTables.contains(storageTableName)) { dropHiveTable(storageTableName); } droppedTables.add(storageTableName); } } // updateFact will be false when fact is fully dropped
private CubeFactTable checkFactStorage(LensSessionHandle sessionid, String fact, String storage) throws HiveException, LensException { CubeMetastoreClient client = getClient(sessionid); CubeFactTable factTable = client.getCubeFactTable(fact); client.verifyStorageExists(factTable, storage); return factTable; }
@Override public void dropAllStoragesOfFact(LensSessionHandle sessionid, String factName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); CubeFactTable tab = msClient.getCubeFactTable(factName); int total = tab.getStorages().size(); int i = 0; List<String> storageNames = new ArrayList<>(tab.getStorages()); for (String s : storageNames) { msClient.dropStorageFromFact(factName, s); log.info("Dropped storage " + s + " from fact table " + factName + " [" + ++i + "/" + total + "]"); } log.info("Dropped " + total + " storages from fact table " + factName); } }
private Set<String> getAllTablesForStorage(LensSessionHandle sessionHandle, String fact, String storageName) throws LensException { Set<String> storageTableNames = new HashSet<>(); if (getClient(sessionHandle).isFactTable(fact)) { CubeFactTable cft = getClient(sessionHandle).getCubeFactTable(fact); Map<UpdatePeriod, String> storageMap = cft.getStoragePrefixUpdatePeriodMap().get(storageName); for (Map.Entry entry : storageMap.entrySet()) { storageTableNames.add(MetastoreUtil.getStorageTableName(fact, Storage.getPrefix((String) entry.getValue()))); } } else { storageTableNames.add(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName)); } return storageTableNames; }
private void updateFactCache(String factTableName) throws LensException { if (enableCaching) { Table factTbl = getTableWithTypeFailFast(factTableName, CubeTableType.FACT); FactTable refreshedTable; if (factTbl.getParameters().get(getSourceFactNameKey(factTableName)) != null) { String sourceFactName = factTbl.getParameters().get(getSourceFactNameKey(factTableName)); refreshedTable = new CubeVirtualFactTable(refreshTable(factTableName), getCubeFactTable(sourceFactName)); } else { refreshedTable = new CubeFactTable(refreshTable(factTableName)); } allFactTables.put(factTableName.trim().toLowerCase(), refreshedTable); } }
private void updateFactCache(String factTableName) throws LensException { if (enableCaching) { Table factTbl = getTableWithTypeFailFast(factTableName, CubeTableType.FACT); FactTable refreshedTable; if (factTbl.getParameters().get(getSourceFactNameKey(factTableName)) != null){ String sourceFactName = factTbl.getParameters().get(getSourceFactNameKey(factTableName)); refreshedTable = new CubeVirtualFactTable(refreshTable(factTableName), getCubeFactTable(sourceFactName)); }else { refreshedTable = new CubeFactTable(refreshTable(factTableName)); } allFactTables.put(factTableName.trim().toLowerCase(), refreshedTable); } }
private FactTable getFactTable(Table tbl) throws LensException { String sourceFact = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())); if (sourceFact != null) { return new CubeVirtualFactTable(tbl, getCubeFactTable(sourceFact)); } else { return new CubeFactTable(tbl); } }
private FactTable getFactTable(Table tbl) throws LensException { String sourceFact = tbl.getParameters().get(getSourceFactNameKey(tbl.getTableName())); if (sourceFact != null) { return new CubeVirtualFactTable(tbl, getCubeFactTable(sourceFact)); } else { return new CubeFactTable(tbl); } }
private void createSummaryPartitions(CubeMetastoreClient client) throws Exception { String factName = "summary1"; CubeFactTable fact1 = client.getCubeFactTable(factName); createPIEParts(client, fact1, c2); factName = "summary2"; CubeFactTable fact2 = client.getCubeFactTable(factName); createPIEParts(client, fact2, c2); factName = "summary3"; CubeFactTable fact3 = client.getCubeFactTable(factName); createPIEParts(client, fact3, c2); factName = "summary4"; CubeFactTable fact4 = client.getCubeFactTable(factName); createPIEParts(client, fact4, c2); factName = "summary5"; CubeFactTable fact5 = client.getCubeFactTable(factName); createPIParts(client, fact5, c98); }
private void createBaseCubeFactPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "testFact5_RAW_BASE"; CubeFactTable fact = client.getCubeFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); Date temp = cal.getTime(); while (!(temp.after(NOW))) { Map<String, Date> timeParts = new HashMap<String, Date>(); timeParts.put("dt", temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, c1, CubeTableType.FACT); cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } }
private void dropStorageFromFact(String factName, String storage, boolean updateFact) throws LensException { dropHiveTablesForStorage(factName, storage); if (updateFact) { CubeFactTable cft = getCubeFactTable(factName); cft.dropStorage(storage); alterCubeTable(factName, getTableWithTypeFailFast(factName, CubeTableType.FACT), cft); updateFactCache(factName); } }
@Override public List<String> getStoragesOfFact(LensSessionHandle sessionid, String fact) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); if (!msClient.isFactTable(fact)) { throw new NotFoundException("Not a fact table " + fact); } CubeFactTable cft = msClient.getCubeFactTable(fact); if (cft != null) { return new ArrayList<>(cft.getStorages()); } else { throw new NotFoundException("Could not get fact table " + fact); } } }
private void createTestFact2RawPartitions(CubeMetastoreClient client) throws HiveException, LensException { String factName = "testFact2_raw"; CubeFactTable fact2 = client.getCubeFactTable(factName); // Add all hourly partitions for two days Calendar cal = Calendar.getInstance(); cal.setTime(TWODAYS_BACK); Date temp = cal.getTime(); while (!(temp.after(NOW))) { Map<String, Date> timeParts = new HashMap<String, Date>(); timeParts.put(TestCubeMetastoreClient.getDatePartitionKey(), temp); StoragePartitionDesc sPartSpec = new StoragePartitionDesc(fact2.getName(), timeParts, null, HOURLY); client.addPartition(sPartSpec, c3, CubeTableType.FACT); cal.add(HOUR_OF_DAY, 1); temp = cal.getTime(); } }
private void dropStorageFromFact(String factName, String storage, boolean updateFact) throws LensException { dropHiveTablesForStorage(factName, storage); if (updateFact) { CubeFactTable cft = getCubeFactTable(factName); cft.dropStorage(storage); alterCubeTable(factName, getTableWithTypeFailFast(factName, CubeTableType.FACT), cft); updateFactCache(factName); } }
/** * Drop a storage from fact * * @param factName fact name * @param storage storage name * @throws LensException */ public void dropStorageFromFact(String factName, String storage) throws LensException { CubeFactTable cft = getCubeFactTable(factName); dropHiveTablesForStorage(factName, storage); cft.dropStorage(storage); alterCubeTable(factName, getTableWithTypeFailFast(factName, CubeTableType.FACT), cft); updateFactCache(factName); }
@Override public void addStorageToFact(LensSessionHandle sessionid, String fact, XStorageTableElement storageTable) throws LensException { Set<UpdatePeriod> updatePeriods = new TreeSet<>(); for (XUpdatePeriod sup : storageTable.getUpdatePeriods().getUpdatePeriod()) { updatePeriods.add(UpdatePeriod.valueOf(sup.name())); } try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient msClient = getClient(sessionid); XStorageTables tables = new XStorageTables(); tables.getStorageTable().add(storageTable); msClient.addStorage(msClient.getCubeFactTable(fact), storageTable.getStorageName(), updatePeriods, JAXBUtils.tableDescPrefixMapFromXStorageTables(tables), JAXBUtils.storageTablePrefixMapOfStorage(tables).get(storageTable.getStorageName())); log.info("Added storage " + storageTable.getStorageName() + ":" + updatePeriods + " for fact " + fact); } }
private String getStorageTablePrefixFromStorage(String factOrDimTableName, String storage, UpdatePeriod updatePeriod) throws LensException { if (updatePeriod == null) { return storage; } if (isVirtualFactTable(factOrDimTableName)) { CubeFactTable sourceFact = (CubeFactTable) getCubeVirtualFactTable(factOrDimTableName).getSourceCubeFactTable(); return sourceFact.getTablePrefix(storage, updatePeriod); } else if (isFactTable(factOrDimTableName)) { return getCubeFactTable(factOrDimTableName).getTablePrefix(storage, updatePeriod); } else { return storage; } }
private String getStorageTablePrefixFromStorage(String factOrDimTableName, String storage, UpdatePeriod updatePeriod) throws LensException { if (updatePeriod == null) { return storage; } if (isVirtualFactTable(factOrDimTableName)) { CubeFactTable sourceFact = (CubeFactTable) getCubeVirtualFactTable(factOrDimTableName).getSourceCubeFactTable(); return sourceFact.getTablePrefix(storage, updatePeriod); } else if (isFactTable(factOrDimTableName)) { return getCubeFactTable(factOrDimTableName).getTablePrefix(storage, updatePeriod); } else { return storage; } }
/** * Drop a storage from fact * * @param factName fact name * @param storage storage name * @throws LensException */ public void dropStorageFromFact(String factName, String storage) throws LensException { CubeFactTable cft = getCubeFactTable(factName); checkIfAuthorized(); dropHiveTablesForStorage(factName, storage); cft.dropStorage(storage); alterCubeTable(factName, getTableWithTypeFailFast(factName, CubeTableType.FACT), cft); updateFactCache(factName); }