@Override public String getSourceFactName() { return this.sourceCubeFactTable.getName(); } }
@Override public String getSourceFactName() { return this.sourceCubeFactTable.getName(); }
private void updateAllVirtualFacts(FactTable cubeFactTable) throws LensException { if (enableCaching) { String cubeFactTableName = cubeFactTable.getName().trim().toLowerCase(); if (factToVirtualFactMapping.get(cubeFactTableName) != null) { synchronized (allFactTables) { List<String> virtualFactTableNames = factToVirtualFactMapping.get(cubeFactTableName); for (String vf : virtualFactTableNames) { CubeVirtualFactTable cvf = getCubeVirtualFactTable(vf); cvf.setSourceCubeFactTable(cubeFactTable); allFactTables.put(vf.trim().toLowerCase(), cvf); } } } } }
private void updateAllVirtualFacts(FactTable cubeFactTable) throws LensException { if (enableCaching) { String cubeFactTableName = cubeFactTable.getName().trim().toLowerCase(); if (factToVirtualFactMapping.get(cubeFactTableName) != null) { synchronized (allFactTables) { List<String> virtualFactTableNames = factToVirtualFactMapping.get(cubeFactTableName); for (String vf : virtualFactTableNames) { CubeVirtualFactTable cvf = getCubeVirtualFactTable(vf); cvf.setSourceCubeFactTable(cubeFactTable); allFactTables.put(vf.trim().toLowerCase(), cvf); } } } } }
/** extract storage name from fact and storage table name. String operation */ private String extractStorageName(FactTable fact, String storageTableName) throws LensException { int ind = storageTableName.lastIndexOf(fact.getSourceFactName()); if (ind <= 0) { throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); } String name = storageTableName.substring(0, ind - StorageConstants.STORGAE_SEPARATOR.length()); for (String storageName : fact.getStorages()) { if (name.equalsIgnoreCase(storageName)) { return storageName; } } throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); }
/** extract storage name from fact and storage table name. String operation */ private String extractStorageName(FactTable fact, String storageTableName) throws LensException { int ind = storageTableName.lastIndexOf(fact.getSourceFactName()); if (ind <= 0) { throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); } String name = storageTableName.substring(0, ind - StorageConstants.STORGAE_SEPARATOR.length()); for (String storageName : fact.getStorages()) { if (name.equalsIgnoreCase(storageName)) { return storageName; } } throw new LensException("storageTable: " + storageTableName + ", does not belong to fact: " + fact.getName()); }
/** * Drop a fact with cascade flag * * @param factName fact name * @param cascade If true, will drop all the storages of the fact * @throws LensException */ public void dropFact(String factName, boolean cascade) throws LensException { getTableWithTypeFailFast(factName, CubeTableType.FACT); FactTable fact = getFactTable(factName); if (cascade) { for (String storage : fact.getStorages()) { dropStorageFromFact(factName, storage, false); } } dropHiveTable(factName); allFactTables.remove(factName.trim().toLowerCase()); if (fact.isVirtualFact()) { String sourceFactTable = fact.getProperties().get(getSourceFactNameKey(fact.getName())); if (factToVirtualFactMapping.get(sourceFactTable) != null && factToVirtualFactMapping.get(sourceFactTable).contains(fact.getName())) { factToVirtualFactMapping.get(sourceFactTable).remove(fact.getName()); } } else { dropAllVirtualFactTables(factName); } }
@Override public List<String> getAllFactNames(LensSessionHandle sessionid, String cubeName) throws LensException { try (SessionContext ignored = new SessionContext(sessionid)){ CubeMetastoreClient client = getClient(sessionid); CubeInterface fact = client.getCube(cubeName); if (cubeName != null && fact == null) { throw new LensException("Could not get table: " + cubeName + " as a cube"); } Collection<FactTable> facts = client.getAllFacts(fact); List<String> factNames = new ArrayList<>(facts.size()); for (FactTable cft : facts) { factNames.add(cft.getName()); } return factNames; } }
/** * Drop a fact with cascade flag * * @param factName fact name * @param cascade If true, will drop all the storages of the fact * @throws LensException */ public void dropFact(String factName, boolean cascade) throws LensException { getTableWithTypeFailFast(factName, CubeTableType.FACT); FactTable fact = getFactTable(factName); checkIfAuthorized(); if (cascade) { for (String storage : fact.getStorages()) { dropStorageFromFact(factName, storage, false); } } dropHiveTable(factName); allFactTables.remove(factName.trim().toLowerCase()); if (fact.isVirtualFact()) { String sourceFactTable = fact.getProperties().get(getSourceFactNameKey(fact.getName())); if (factToVirtualFactMapping.get(sourceFactTable) != null && factToVirtualFactMapping.get(sourceFactTable).contains(fact.getName())) { factToVirtualFactMapping.get(sourceFactTable).remove(fact.getName()); } } else { dropAllVirtualFactTables(factName); } }
@Override protected void addProperties() { getProperties().put(MetastoreConstants.TABLE_TYPE_KEY, getTableType().name()); getProperties().put(MetastoreUtil.getSourceFactNameKey(this.getName()), this.sourceCubeFactTable.getName()); if (virtualFactWeight.isPresent()) { getProperties().put(MetastoreUtil.getCubeTableWeightKey(this.getName()), String.valueOf(virtualFactWeight.get())); } this.getProperties().put(MetastoreUtil.getFactCubeNameKey(getName()), cubeName); }
@Override protected void addProperties() { getProperties().put(MetastoreConstants.TABLE_TYPE_KEY, getTableType().name()); getProperties().put(MetastoreUtil.getSourceFactNameKey(this.getName()), this.sourceCubeFactTable.getName()); if (virtualFactWeight.isPresent()) { getProperties().put(MetastoreUtil.getCubeTableWeightKey(this.getName()), String.valueOf(virtualFactWeight.get())); } this.getProperties().put(MetastoreUtil.getFactCubeNameKey(getName()), cubeName); }
FactTable fact = getFactTable(table, false); if (fact != null) { if (fact.getProperties().get(getSourceFactNameKey(fact.getName())) != null) { //is virtual fact if (includeVirtualFacts) { facts.add(fact);
FactTable fact = getFactTable(table, false); if (fact != null) { if (fact.getProperties().get(getSourceFactNameKey(fact.getName())) != null) { //is virtual fact if (includeVirtualFacts) { facts.add(fact);
public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, CubeQueryContext cubeQueryContext) throws LensException { this.cube = cube; this.fact = fact; this.cubeQueryContext = cubeQueryContext; if ((getCube() == null) || (fact == null) || (storageName == null)) { throw new IllegalArgumentException("Cube,fact and storageName should be non null"); } this.storageName = storageName; this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(fact.getSourceFactName(), storageName); this.name = getFact().getName(); this.processTimePartCol = getConf().get(CubeQueryConfUtil.PROCESS_TIME_PART_COL); String formatStr = getConf().get(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT); if (formatStr != null) { this.partWhereClauseFormat = new SimpleDateFormat(formatStr); } completenessPartCol = getConf().get(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL); completenessThreshold = getConf() .getFloat(CubeQueryConfUtil.COMPLETENESS_THRESHOLD, CubeQueryConfUtil.DEFAULT_COMPLETENESS_THRESHOLD); Set<String> storageTblNames = getCubeMetastoreClient().getStorageTables(fact, storageName); isStorageTblsAtUpdatePeriodLevel = storageTblNames.size() > 1 || !storageTblNames.iterator().next().equalsIgnoreCase(storageTable); setStorageStartAndEndDate(); }
public StorageCandidate(CubeInterface cube, FactTable fact, String storageName, CubeQueryContext cubeQueryContext) throws LensException { this.cube = cube; this.fact = fact; this.cubeQueryContext = cubeQueryContext; if ((getCube() == null) || (fact == null) || (storageName == null)) { throw new IllegalArgumentException("Cube,fact and storageName should be non null"); } this.storageName = storageName; this.storageTable = MetastoreUtil.getFactOrDimtableStorageTableName(fact.getSourceFactName(), storageName); this.name = getFact().getName(); this.processTimePartCol = getConf().get(CubeQueryConfUtil.PROCESS_TIME_PART_COL); String formatStr = getConf().get(CubeQueryConfUtil.PART_WHERE_CLAUSE_DATE_FORMAT); if (formatStr != null) { this.partWhereClauseFormat = new SimpleDateFormat(formatStr); } completenessPartCol = getConf().get(CubeQueryConfUtil.COMPLETENESS_CHECK_PART_COL); completenessThreshold = getConf() .getFloat(CubeQueryConfUtil.COMPLETENESS_THRESHOLD, CubeQueryConfUtil.DEFAULT_COMPLETENESS_THRESHOLD); Set<String> storageTblNames = getCubeMetastoreClient().getStorageTables(fact, storageName); isStorageTblsAtUpdatePeriodLevel = storageTblNames.size() > 1 || !storageTblNames.iterator().next().equalsIgnoreCase(storageTable); setStorageStartAndEndDate(); }
log.info("Not considering fact: {} as it has no update periods", fact.getName()); } else if (validFactTables != null && !validFactTables.contains(fact.getName())) { log.info("Not considering fact: {} as it's not valid as per user configuration.", fact.getName()); } else { for (String s : fact.getStorages()) {
public static XVirtualFactTable virtualFactTableFromVirtualCubeFactTable(CubeVirtualFactTable vFact) { XVirtualFactTable fact = XCF.createXVirtualFactTable(); fact.setName(vFact.getName()); fact.setProperties(new XProperties()); fact.getProperties().getProperty().addAll(xPropertiesFromMap(vFact.getProperties())); fact.setWeight(vFact.weight()); fact.setSourceFactName(vFact.getSourceCubeFactTable().getName()); fact.setCubeName(vFact.getCubeName()); return fact; }
public static XFactTable factTableFromCubeFactTable(FactTable cFact) { XFactTable fact = XCF.createXFactTable(); fact.setName(cFact.getName()); fact.setColumns(new XColumns()); fact.setProperties(new XProperties()); fact.setStorageTables(new XStorageTables()); fact.getProperties().getProperty().addAll(xPropertiesFromMap(cFact.getProperties())); fact.getColumns().getColumn().addAll(columnsFromFieldSchemaList(cFact.getColumns())); fact.setWeight(cFact.weight()); fact.setCubeName(cFact.getCubeName()); return fact; }
public static XVirtualFactTable virtualFactTableFromVirtualCubeFactTable(CubeVirtualFactTable vFact) { XVirtualFactTable fact = XCF.createXVirtualFactTable(); fact.setName(vFact.getName()); fact.setProperties(new XProperties()); fact.getProperties().getProperty().addAll(xPropertiesFromMap(vFact.getProperties())); fact.setWeight(vFact.weight()); fact.setSourceFactName(vFact.getSourceCubeFactTable().getName()); fact.setCubeName(vFact.getCubeName()); return fact; }
public static XFactTable factTableFromCubeFactTable(FactTable cFact) { XFactTable fact = XCF.createXFactTable(); fact.setName(cFact.getName()); fact.setColumns(new XColumns()); fact.setProperties(new XProperties()); fact.setStorageTables(new XStorageTables()); fact.getProperties().getProperty().addAll(xPropertiesFromMap(cFact.getProperties())); fact.getColumns().getColumn().addAll(columnsFromFieldSchemaList(cFact.getColumns())); fact.setWeight(cFact.weight()); fact.setCubeName(cFact.getCubeName()); return fact; }