@Override public String getSourceFactName() { return this.sourceCubeFactTable.getName(); } }
@Override public Map<String, Set<UpdatePeriod>> getUpdatePeriods() { return this.sourceCubeFactTable.getUpdatePeriods(); }
public Date getAbsoluteEndTime() { String absoluteEndTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_END_TIME); Date absoluteDate = null; if (StringUtils.isNotBlank(absoluteEndTime)) { absoluteDate = MetastoreUtil.getDateFromProperty(absoluteEndTime, false, false); } return absoluteDate == null ? this.sourceCubeFactTable.getAbsoluteEndTime() : absoluteDate; }
@Override public Collection<String> getColumns() { if (factColumns == null) { factColumns = fact.getValidColumns(); if (factColumns == null) { factColumns = fact.getAllFieldNames(); } } return factColumns; }
@Override public Collection<String> getColumns() { if (factColumns == null) { factColumns = fact.getValidColumns(); if (factColumns == null) { factColumns = fact.getAllFieldNames(); factColumns.addAll(fact.getPartitionColumns(storageName)); } } return factColumns; }
/** extract storage name from fact and storage table name. String operation */ private String extractStorageName(FactTable fact, String storageTableName) throws LensException { int ind = storageTableName.lastIndexOf(fact.getSourceFactName()); if (ind <= 0) { throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); } String name = storageTableName.substring(0, ind - StorageConstants.STORGAE_SEPARATOR.length()); for (String storageName : fact.getStorages()) { if (name.equalsIgnoreCase(storageName)) { return storageName; } } throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); }
public static XFactTable factTableFromCubeFactTable(FactTable cFact) { XFactTable fact = XCF.createXFactTable(); fact.setName(cFact.getName()); fact.setColumns(new XColumns()); fact.setProperties(new XProperties()); fact.setStorageTables(new XStorageTables()); fact.getProperties().getProperty().addAll(xPropertiesFromMap(cFact.getProperties())); fact.getColumns().getColumn().addAll(columnsFromFieldSchemaList(cFact.getColumns())); fact.setWeight(cFact.weight()); fact.setCubeName(cFact.getCubeName()); return fact; }
/** * Drop a fact with cascade flag * * @param factName fact name * @param cascade If true, will drop all the storages of the fact * @throws LensException */ public void dropFact(String factName, boolean cascade) throws LensException { getTableWithTypeFailFast(factName, CubeTableType.FACT); FactTable fact = getFactTable(factName); if (cascade) { for (String storage : fact.getStorages()) { dropStorageFromFact(factName, storage, false); } } dropHiveTable(factName); allFactTables.remove(factName.trim().toLowerCase()); if (fact.isVirtualFact()) { String sourceFactTable = fact.getProperties().get(getSourceFactNameKey(fact.getName())); if (factToVirtualFactMapping.get(sourceFactTable) != null && factToVirtualFactMapping.get(sourceFactTable).contains(fact.getName())) { factToVirtualFactMapping.get(sourceFactTable).remove(fact.getName()); } } else { dropAllVirtualFactTables(factName); } }
if (fact.getUpdatePeriods().isEmpty()) { log.info("Not considering fact: {} as it has no update periods", fact.getName()); } else if (validFactTables != null && !validFactTables.contains(fact.getName())) { log.info("Not considering fact: {} as it's not valid as per user configuration.", fact.getName()); } else { for (String s : fact.getStorages()) { StorageCandidate sc = new StorageCandidate(cubeql.getCube(), fact, s, cubeql); if (isStorageSupportedOnDriver(sc.getStorageName())) {
StorageCandidate sc = (StorageCandidate) c; String str = conf.get(CubeQueryConfUtil.getValidStorageTablesKey(sc.getFact().getName())); List<String> validFactStorageTables = StringUtils.isBlank(str) ? null : Arrays.asList(StringUtils.split(str.toLowerCase(), ",")); .getStringList(conf, CubeQueryConfUtil.getValidUpdatePeriodsKey(sc.getFact().getName(), sc.getStorageName())); boolean isUpdatePeriodForStorageAdded = false; Map<String, SkipUpdatePeriodCode> skipUpdatePeriodCauses = new HashMap<>(); for (UpdatePeriod updatePeriod : sc.getFact().getUpdatePeriods().get(sc.getStorageName())) { if (maxInterval != null && updatePeriod.compareTo(maxInterval) > 0) { if (!client.partColExists(sc.getFact(), sc.getStorageName(), range.getPartitionColumn())) { pruningCauseForThisTimeRange = partitionColumnsMissing(range.getPartitionColumn()); TimeRange fallBackRange = getFallbackRange(range, sc.getFact().getSourceFactName(), cubeql); while (fallBackRange != null) { pruningCauseForThisTimeRange = null; fallBackRange.getPartitionColumn())) { pruningCauseForThisTimeRange = partitionColumnsMissing(fallBackRange.getPartitionColumn()); fallBackRange = getFallbackRange(fallBackRange, sc.getFact().getSourceFactName(), cubeql); } else { if (!sc.isPartiallyValidForTimeRange(fallBackRange)) {
public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, CubeQueryContext cubeQueryContext) throws LensException { this.cube = cube; this.fact = fact; this.cubeQueryContext = cubeQueryContext; if ((getCube() == null) || (fact == null) || (storageName == null)) { throw new IllegalArgumentException("Cube,fact and storageName should be non null"); } this.storageName = storageName; this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(fact.getSourceFactName(), storageName); this.name = getFact().getName(); this.processTimePartCol = getConf().get(CubeQueryConfUtil.PROCESS_TIME_PART_COL); String formatStr = getConf().get(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT); if (formatStr != null) { this.partWhereClauseFormat = new SimpleDateFormat(formatStr); } completenessPartCol = getConf().get(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL); completenessThreshold = getConf() .getFloat(CubeQueryConfUtil.COMPLETENESS_THRESHOLD, CubeQueryConfUtil.DEFAULT_COMPLETENESS_THRESHOLD); Set<String> storageTblNames = getCubeMetastoreClient().getStorageTables(fact, storageName); isStorageTblsAtUpdatePeriodLevel = storageTblNames.size() > 1 || !storageTblNames.iterator().next().equalsIgnoreCase(storageTable); setStorageStartAndEndDate(); }
FactTable fact = getFactTable(table, false); if (fact != null) { if (fact.getProperties().get(getSourceFactNameKey(fact.getName())) != null) { //is virtual fact if (includeVirtualFacts) { facts.add(fact);
boolean updated = false; for (FactTable fact : getAllFacts(cube)) { for (String storage : fact.getStorages()) { for (UpdatePeriod updatePeriod : fact.getUpdatePeriods().get(storage)) { PartitionTimeline timeline = partitionTimelineCache.get(fact.getSourceFactName(), storage, updatePeriod, partCol); if (timeline != null) {// this storage table is partitioned by partCol or not.
public XFact getXFactTable(FactTable ft) throws LensException { if (ft.isVirtualFact()) { CubeVirtualFactTable cvft = (CubeVirtualFactTable) ft; XVirtualFactTable factTable = JAXBUtils.virtualFactTableFromVirtualCubeFactTable(cvft); factTable.setSourceFactName(cvft.getSourceCubeFactTable().getName()); fact = factTable; } else {
/** * Returns storage table names for a storage. * Note: If each update period in the storage has a different storage table, this method will return N Storage Tables * where N is the number of update periods in the storage (LENS-1386) * * @param factTable * @param storage * @return * @throws LensException */ public Set<String> getStorageTables(FactTable factTable, String storage) throws LensException { Set<String> uniqueStorageTables = new HashSet<>(); for (UpdatePeriod updatePeriod : factTable.getUpdatePeriods().get(storage)) { String factName = factTable.getSourceFactName(); uniqueStorageTables.add(getStorageTableName(factName, storage, updatePeriod)); } return uniqueStorageTables; }
@Override public String getDataCompletenessTag() { return this.sourceCubeFactTable.getDataCompletenessTag(); }
/** * Get all fact tables of the cube. * * @param cube Cube object * @return List of fact tables * @throws LensException */ public List<FactTable> getAllFacts(CubeInterface cube) throws LensException { String cubeName = null; if (cube != null) { if (cube instanceof DerivedCube) { cube = ((DerivedCube) cube).getParent(); } cubeName = cube.getName(); } List<FactTable> cubeFacts = new ArrayList<>(); for (FactTable fact : getAllFacts()) { if (cubeName == null || fact.getCubeName().equalsIgnoreCase(cubeName)) { cubeFacts.add(fact); } } return cubeFacts; }
public Date getAbsoluteStartTime() { String absoluteStartTime = this.getProperties().get(MetastoreConstants.FACT_ABSOLUTE_START_TIME); Date absoluteDate = null; if (StringUtils.isNotBlank(absoluteStartTime)) { absoluteDate = MetastoreUtil.getDateFromProperty(absoluteStartTime, false, true); } return absoluteDate == null ? this.sourceCubeFactTable.getAbsoluteStartTime() : absoluteDate; }
public Date getStorageTableEndDate(String storageTable, FactTable factTableName) throws LensException { List<Date> endDates = getStorageTimes(storageTable, MetastoreUtil.getStoragetableEndTimesKey()); endDates.add(factTableName.getEndTime()); return Collections.min(endDates); }
@Override public List<FieldSchema> getColumns() { return this.sourceCubeFactTable.getColumns(); }