public String getStorageTableName(String factOrDimTableName, String storage, UpdatePeriod updatePeriod) throws LensException { return MetastoreUtil.getFactOrDimtableStorageTableName(factOrDimTableName, getStorageTablePrefixFromStorage(factOrDimTableName, storage, updatePeriod)); } }
public String getStorageTableName(String factOrDimTableName, String storage, UpdatePeriod updatePeriod) throws LensException { return MetastoreUtil.getFactOrDimtableStorageTableName(factOrDimTableName, getStorageTablePrefixFromStorage(factOrDimTableName, storage, updatePeriod)); } }
/** * Update existing partition * @param client hive client instance * @param fact fact name * @param partition partition to be updated * @throws InvalidOperationException * @throws HiveException */ public void updatePartition(Hive client, String fact, Partition partition) throws InvalidOperationException, HiveException { client.alterPartition(MetastoreUtil.getFactOrDimtableStorageTableName(fact, getName()), partition, null); }
/** * Update existing partition * @param client hive client instance * @param fact fact name * @param partition partition to be updated * @throws InvalidOperationException * @throws HiveException */ public void updatePartition(Hive client, String fact, Partition partition) throws InvalidOperationException, HiveException { client.alterPartition(MetastoreUtil.getFactOrDimtableStorageTableName(fact, getName()), partition, null); }
boolean dimPartitionExists(String dimTblName, String storageName, Map<String, Date> partitionTimestamps) throws HiveException, LensException { String storageTableName = getFactOrDimtableStorageTableName(dimTblName, storageName); return partitionExists(storageTableName, getDimensionTable(dimTblName).getSnapshotDumpPeriods().get(storageName), partitionTimestamps); }
boolean dimPartitionExists(String dimTblName, String storageName, Map<String, Date> partitionTimestamps) throws HiveException, LensException { String storageTableName = getFactOrDimtableStorageTableName(dimTblName, storageName); return partitionExists(storageTableName, getDimensionTable(dimTblName).getSnapshotDumpPeriods().get(storageName), partitionTimestamps); }
private Set<String> getAllTablesForStorage(LensSessionHandle sessionHandle, String fact, String storageName) throws LensException { Set<String> storageTableNames = new HashSet<>(); if (getClient(sessionHandle).isFactTable(fact)) { CubeFactTable cft = getClient(sessionHandle).getCubeFactTable(fact); Map<UpdatePeriod, String> storageMap = cft.getStoragePrefixUpdatePeriodMap().get(storageName); for (Map.Entry entry : storageMap.entrySet()) { storageTableNames.add(MetastoreUtil.getStorageTableName(fact, Storage.getPrefix((String) entry.getValue()))); } } else { storageTableNames.add(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName)); } return storageTableNames; }
private void dropStorageFromDim(String dimTblName, String storage, boolean updateDimTbl) throws LensException { CubeDimensionTable cdt = getDimensionTable(dimTblName); String storageTableName = getFactOrDimtableStorageTableName(dimTblName, storage); dropHiveTable(storageTableName); latestLookupCache.remove(storageTableName.trim().toLowerCase()); if (updateDimTbl) { cdt.dropStorage(storage); alterCubeTable(dimTblName, getTableWithTypeFailFast(dimTblName, CubeTableType.DIM_TABLE), cdt); updateDimCache(dimTblName); } }
private void assertTimeline(CubeMetastoreClient client, String factName, String storageName, UpdatePeriod updatePeriod, String timeDim, Class<? extends PartitionTimeline> partitionTimelineClass) throws Exception { String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, storageName); PartitionTimeline expectedTimeline = partitionTimelineClass.getConstructor( String.class, UpdatePeriod.class, String.class) .newInstance(storageTableName, updatePeriod, timeDim); assertTimeline(client, factName, storageName, updatePeriod, timeDim, expectedTimeline); }
private void dropStorageFromDim(String dimTblName, String storage, boolean updateDimTbl) throws LensException { checkIfAuthorized(); CubeDimensionTable cdt = getDimensionTable(dimTblName); String storageTableName = getFactOrDimtableStorageTableName(dimTblName, storage); dropHiveTable(storageTableName); latestLookupCache.remove(storageTableName.trim().toLowerCase()); if (updateDimTbl) { cdt.dropStorage(storage); alterCubeTable(dimTblName, getTableWithTypeFailFast(dimTblName, CubeTableType.DIM_TABLE), cdt); updateDimCache(dimTblName); } }
for (String storageName : dimTable.getStorages()) { if (dimTable.hasStorageSnapshots(storageName)) { String storageTableName = getFactOrDimtableStorageTableName(dimTable.getName(), storageName); if (dimTableLatestPartitionExistsInMetastore(storageTableName,
for (String storageName : dimTable.getStorages()) { if (dimTable.hasStorageSnapshots(storageName)) { String storageTableName = getFactOrDimtableStorageTableName(dimTable.getName(), storageName); if (dimTableLatestPartitionExistsInMetastore(storageTableName,
public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, CubeQueryContext cubeQueryContext) throws LensException { this.cube = cube; this.fact = fact; this.cubeQueryContext = cubeQueryContext; if ((getCube() == null) || (fact == null) || (storageName == null)) { throw new IllegalArgumentException("Cube,fact and storageName should be non null"); } this.storageName = storageName; this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(fact.getSourceFactName(), storageName); this.name = getFact().getName(); this.processTimePartCol = getConf().get(CubeQueryConfUtil.PROCESS_TIME_PART_COL); String formatStr = getConf().get(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT); if (formatStr != null) { this.partWhereClauseFormat = new SimpleDateFormat(formatStr); } completenessPartCol = getConf().get(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL); completenessThreshold = getConf() .getFloat(CubeQueryConfUtil.COMPLETENESS_THRESHOLD, CubeQueryConfUtil.DEFAULT_COMPLETENESS_THRESHOLD); Set<String> storageTblNames = getCubeMetastoreClient().getStorageTables(fact, storageName); isStorageTblsAtUpdatePeriodLevel = storageTblNames.size() > 1 || !storageTblNames.iterator().next().equalsIgnoreCase(storageTable); setStorageStartAndEndDate(); }
public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, CubeQueryContext cubeQueryContext) throws LensException { this.cube = cube; this.fact = fact; this.cubeQueryContext = cubeQueryContext; if ((getCube() == null) || (fact == null) || (storageName == null)) { throw new IllegalArgumentException("Cube,fact and storageName should be non null"); } this.storageName = storageName; this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(fact.getSourceFactName(), storageName); this.name = getFact().getName(); this.processTimePartCol = getConf().get(CubeQueryConfUtil.PROCESS_TIME_PART_COL); String formatStr = getConf().get(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT); if (formatStr != null) { this.partWhereClauseFormat = new SimpleDateFormat(formatStr); } completenessPartCol = getConf().get(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL); completenessThreshold = getConf() .getFloat(CubeQueryConfUtil.COMPLETENESS_THRESHOLD, CubeQueryConfUtil.DEFAULT_COMPLETENESS_THRESHOLD); Set<String> storageTblNames = getCubeMetastoreClient().getStorageTables(fact, storageName); isStorageTblsAtUpdatePeriodLevel = storageTblNames.size() > 1 || !storageTblNames.iterator().next().equalsIgnoreCase(storageTable); setStorageStartAndEndDate(); }
private void assertTimeline(CubeMetastoreClient client, String factName, String storageName, UpdatePeriod updatePeriod, String timeDim, PartitionTimeline expectedTimeline) throws Exception { assertNotNull(factName); assertNotNull(storageName); assertNotNull(updatePeriod); assertNotNull(timeDim); String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(factName, storageName); List<PartitionTimeline> timelines = client.getTimelines(factName, storageName, updatePeriod.name(), timeDim); assertEquals(timelines.size(), 1); PartitionTimeline actualTimeline = timelines.get(0); assertEquals(actualTimeline, expectedTimeline); assertEquals(client.getTable(storageTableName).getParameters() .get(MetastoreUtil.getPartitionTimelineStorageClassKey(updatePeriod, timeDim)), expectedTimeline.getClass().getCanonicalName()); expectedTimeline.init(client.getTable(MetastoreUtil.getFactOrDimtableStorageTableName(factName, storageName))); assertEquals(actualTimeline, expectedTimeline); }
public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); if (!dimTable.getStorages().isEmpty()) { for (String storageName : dimTable.getStorages()) { XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } dt.getStorageTables().getStorageTable().add(tblElement); } } return dt; }
public XDimensionTable getXDimensionTable(CubeDimensionTable dimTable) throws LensException { XDimensionTable dt = JAXBUtils.dimTableFromCubeDimTable(dimTable); if (!dimTable.getStorages().isEmpty()) { for (String storageName : dimTable.getStorages()) { XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTable.getName(), storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } dt.getStorageTables().getStorageTable().add(tblElement); } } return dt; } /**
public XStorageTableElement getStorageOfFact(LensSessionHandle sessionid, String fact, String storageName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)) { CubeMetastoreClient msClient = getClient(sessionid); FactTable factTable = msClient.getFactTable(fact); Set<UpdatePeriod> updatePeriods = factTable.getUpdatePeriods().get(storageName); XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(fact, storageName))); tblElement.setStorageName(storageName); for (UpdatePeriod p : updatePeriods) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } return tblElement; } }
public XStorageTableElement getStorageOfDim(LensSessionHandle sessionid, String dimTblName, String storageName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient msClient = getClient(sessionid); CubeDimensionTable dimTable = msClient.getDimensionTable(dimTblName); XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable( msClient.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(dimTblName, storageName))); tblElement.setStorageName(storageName); UpdatePeriod p = dimTable.getSnapshotDumpPeriods().get(storageName); if (p != null) { tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name())); } return tblElement; } }
@Override public XPartitionList getAllPartitionsOfDimTableStorage( LensSessionHandle sessionid, String dimTable, String storageName, String filter) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ checkDimTableStorage(sessionid, dimTable, storageName); CubeMetastoreClient client = getClient(sessionid); String storageTableName = MetastoreUtil.getFactOrDimtableStorageTableName(dimTable, storageName); List<Partition> partitions = client.getPartitionsByFilter(storageTableName, filter); List<String> timePartCols = client.getTimePartColNamesOfTable(storageTableName); return xpartitionListFromPartitionList(dimTable, partitions, timePartCols); } catch (HiveException exc) { throw new LensException(exc); } }