public boolean isFactTable(String fullTableName) { for (TableRef t : factTableRefs) { if (t.getTableIdentity().equals(fullTableName)) return true; } return false; }
public boolean isLookupTable(String fullTableName) { for (TableRef t : lookupTableRefs) { if (t.getTableIdentity().equals(fullTableName)) return true; } return false; }
public TableRef findFirstTable(String tableIdentity) throws IllegalArgumentException { if (rootFactTableRef.getTableIdentity().equals(tableIdentity)) return rootFactTableRef; for (TableRef fact : factTableRefs) { if (fact.getTableIdentity().equals(tableIdentity)) return fact; } for (TableRef lookup : lookupTableRefs) { if (lookup.getTableIdentity().equals(tableIdentity)) return lookup; } throw new IllegalArgumentException("Table not found by " + tableIdentity + " in model " + name); }
private Set<String> getAllTblsWithSchema(List<OLAPContext> contexts) { // all tables with DB, Like DB.TABLE Set<String> tableWithSchema = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (OLAPContext context : contexts) { for (OLAPTableScan tableScan : context.allTableScans) { tableWithSchema.add(tableScan.getTableRef().getTableIdentity()); } } return tableWithSchema; }
public DimensionDesc findDimensionByTable(String lookupTableName) { lookupTableName = lookupTableName.toUpperCase(Locale.ROOT); for (DimensionDesc dim : dimensions) if (dim.getTableRef() != null && dim.getTableRef().getTableIdentity().equals(lookupTableName)) return dim; return null; }
private List<String> getCompatibleTablePath(Set<TableRef> tableRefs, String project, String rootPath) throws IOException { List<String> toResource = new ArrayList<>(); List<String> paths = srcStore.collectResourceRecursively(rootPath, MetadataConstants.FILE_SURFIX); Map<String, String> tableMap = new HashMap<>(); for (String path : paths) for (TableRef tableRef : tableRefs) { String tableId = tableRef.getTableIdentity(); if (path.contains(tableId)) { String prj = TableDesc.parseResourcePath(path).getSecond(); if (prj == null && tableMap.get(tableId) == null) tableMap.put(tableRef.getTableIdentity(), path); if (prj != null && prj.contains(project)) { tableMap.put(tableRef.getTableIdentity(), path); } } } for (Map.Entry<String, String> e : tableMap.entrySet()) { toResource.add(e.getValue()); } return toResource; }
private void addAlias(TableRef ref) { String alias = ref.getAlias(); if (aliasMap.containsKey(alias)) throw new IllegalStateException("Alias '" + alias + "' ref to multiple tables: " + ref.getTableIdentity() + ", " + aliasMap.get(alias).getTableIdentity()); aliasMap.put(alias, ref); TableDesc table = ref.getTableDesc(); addTableName(table.getName(), ref); addTableName(table.getIdentity(), ref); }
public String getRootFactTable() { return getModel().getRootFactTable().getTableIdentity(); }
private boolean belongToTable(FunctionDesc func, String table, DataModelDesc model) { // measure belong to the first column parameter's table List<TblColRef> cols = func.getParameter().getColRefs(); String belongTo = cols.isEmpty() ? model.getRootFactTable().getTableIdentity() : cols.get(0).getTable(); return belongTo.equals(table); }
public static boolean needRewrite(OLAPContext ctx) { if (ctx.hasJoin) return true; if (ctx.realization == null) { return false; } String realRootFact = ctx.realization.getModel().getRootFactTable().getTableIdentity(); if (ctx.firstTableScan.getTableName().equals(realRootFact)) return true; return false; } }
private String getRootFactTableForRealization(IRealization realization) { if (realization == null) { logger.warn("Cannot find realization %s", realization); return null; } DataModelDesc model = realization.getModel(); if (model == null) { logger.warn("The model for realization %s is null", realization.getName()); return null; } TableRef rootFactTable = model.getRootFactTable(); if (rootFactTable == null) { logger.warn("The root table for model %s is null", model.getName()); return null; } return rootFactTable.getTableIdentity(); }
private void updateMeta(KylinConfig config, String projectName, String cubeName, DataModelDesc model) { String[] nodes = config.getRestServers(); Map<String, String> tableToProjects = new HashMap<>(); for (TableRef tableRef : model.getAllTables()) { tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject()); } for (String node : nodes) { RestClient restClient = new RestClient(node); try { logger.info("update meta cache for " + node); restClient.clearCacheForCubeMigration(cubeName, projectName, model.getName(), tableToProjects); } catch (IOException e) { logger.error(e.getMessage()); } } }
public boolean containsTable(TableDesc table) { for (TableRef t : allTableRefs) { if (t.getTableIdentity().equals(table.getIdentity()) && StringUtil.equals(t.getTableDesc().getProject(), table.getProject())) return true; } return false; }
private String[] getLookupKeyColumns(CubeInstance cube, String tableName) { CubeDesc cubeDesc = cube.getDescriptor(); DataModelDesc modelDesc = cubeDesc.getModel(); TableRef lookupTableRef = null; for (TableRef tableRef : modelDesc.getLookupTables()) { if (tableRef.getTableIdentity().equalsIgnoreCase(tableName)) { lookupTableRef = tableRef; break; } } if (lookupTableRef == null) { throw new IllegalStateException("cannot find table in model:" + tableName); } JoinDesc joinDesc = modelDesc.getJoinByPKSide(lookupTableRef); TblColRef[] keyColRefs = joinDesc.getPrimaryKeyColumns(); String[] result = new String[keyColRefs.length]; for (int i = 0; i < keyColRefs.length; i++) { result[i] = keyColRefs[i].getName(); } return result; }
public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) { CubeDesc cubeDesc = cube.getDescriptor(); Set<String> allLookupTables = Sets.newHashSet(); for (DimensionDesc dim : cubeDesc.getDimensions()) { TableRef table = dim.getTableRef(); if (cubeDesc.getModel().isLookupTable(table)) { allLookupTables.add(table.getTableIdentity()); } } List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList(); for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) { if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType()) && allLookupTables.contains(snapshotTableDesc.getTableName())) { addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc); } } }
private static Map<String, String> matches(DataModelDesc model, OLAPContext ctx) { Map<String, String> result = Maps.newHashMap(); TableRef firstTable = ctx.firstTableScan.getTableRef(); Map<String, String> matchUp = null; if (ctx.joins.isEmpty() && model.isLookupTable(firstTable.getTableIdentity())) { // one lookup table String modelAlias = model.findFirstTable(firstTable.getTableIdentity()).getAlias(); matchUp = ImmutableMap.of(firstTable.getAlias(), modelAlias); } else if (ctx.joins.size() != ctx.allTableScans.size() - 1) { // has hanging tables ctx.realizationCheck.addModelIncapableReason(model, RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.MODEL_BAD_JOIN_SEQUENCE)); throw new IllegalStateException("Please adjust the sequence of join tables. " + toErrorMsg(ctx)); } else { // normal big joins if (ctx.joinsTree == null) { ctx.joinsTree = new JoinsTree(firstTable, ctx.joins); } matchUp = ctx.joinsTree.matches(model.getJoinsTree(), result); } if (matchUp == null) { ctx.realizationCheck.addModelIncapableReason(model, RealizationCheck.IncapableReason.create(RealizationCheck.IncapableType.MODEL_UNMATCHED_JOIN)); return null; } result.putAll(matchUp); ctx.realizationCheck.addCapableModel(model, result); return result; }
@Override protected void doSetup(Context context) throws IOException { super.bindCurrentConfiguration(context.getConfiguration()); cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME); tableName = context.getConfiguration().get(BatchConstants.CFG_TABLE_NAME); shardNum = Integer.parseInt(context.getConfiguration().get(BatchConstants.CFG_SHARD_NUM)); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(); CubeManager cubeMgr = CubeManager.getInstance(config); cubeDesc = cubeMgr.getCube(cubeName).getDescriptor(); DataModelDesc modelDesc = cubeDesc.getModel(); TableDesc tableDesc = TableMetadataManager.getInstance(KylinConfig.getInstanceFromEnv()).getTableDesc( tableName, cubeDesc.getProject()); TableRef lookupTableRef = null; for (TableRef tableRef : modelDesc.getLookupTables()) { if (tableRef.getTableIdentity().equalsIgnoreCase(tableName)) { lookupTableRef = tableRef; break; } } JoinDesc joinDesc = modelDesc.getJoinByPKSide(lookupTableRef); TblColRef[] keyColRefs = joinDesc.getPrimaryKeyColumns(); String[] keyColumns = new String[keyColRefs.length]; for (int i = 0; i < keyColRefs.length; i++) { keyColumns[i] = keyColRefs[i].getName(); } encoder = new HBaseLookupRowEncoder(tableDesc, keyColumns, shardNum); Configuration conf = context.getConfiguration(); lookupTableInputFormat = MRUtil.getTableInputFormat(tableDesc, conf.get(BatchConstants.ARG_CUBING_JOB_ID)); }
public ILookupTable getLookupTable(CubeSegment cubeSegment, JoinDesc join) { String tableName = join.getPKSide().getTableIdentity(); CubeDesc cubeDesc = cubeSegment.getCubeDesc(); SnapshotTableDesc snapshotTableDesc = cubeDesc.getSnapshotTableDesc(tableName); if (snapshotTableDesc == null || !snapshotTableDesc.isExtSnapshotTable()) { return getInMemLookupTable(cubeSegment, join, snapshotTableDesc); } else { return getExtLookupTable(cubeSegment, tableName, snapshotTableDesc); } }
private ILookupTable getInMemLookupTable(CubeSegment cubeSegment, JoinDesc join, SnapshotTableDesc snapshotTableDesc) { String tableName = join.getPKSide().getTableIdentity(); String snapshotResPath = getSnapshotResPath(cubeSegment, tableName, snapshotTableDesc); String[] pkCols = join.getPrimaryKey(); try { SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); TableDesc tableDesc = getMetadataManager().getTableDesc(tableName, cubeSegment.getProject()); return LookupProviderFactory.getInMemLookupTable(tableDesc, pkCols, snapshot); } catch (IOException e) { throw new IllegalStateException( "Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e); } }
public boolean belongToFactTableDims(TblColRef tblColRef) { if (!belongToContextTables(tblColRef)) { return false; } KylinConfig kylinConfig = olapSchema.getConfig(); String projectName = olapSchema.getProjectName(); String factTableName = firstTableScan.getOlapTable().getTableName(); Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName, factTableName); for (IRealization real : realizations) { DataModelDesc model = real.getModel(); TblColRef.fixUnknownModel(model, tblColRef.getTableRef().getTableIdentity(), tblColRef); // cannot be a measure column Set<String> metrics = Sets.newHashSet(model.getMetrics()); if (metrics.contains(tblColRef.getIdentity())) { tblColRef.unfixTableRef(); return false; } // must belong to a fact table for (TableRef factTable : model.getFactTables()) { if (factTable.getColumns().contains(tblColRef)) { tblColRef.unfixTableRef(); return true; } } tblColRef.unfixTableRef(); } return false; }