|| cube.getSnapshots().containsKey(digest.factTable)) { Set<TblColRef> dimCols = Sets.newHashSet(cube.getModel().findFirstTable(digest.factTable).getColumns());
public static void prepareTestDataForStreamingCube(long startTime, long endTime, int numberOfRecords, String cubeName, StreamDataLoader streamDataLoader) throws IOException { CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName); List<String> data = StreamingTableDataGenerator.generate(numberOfRecords, startTime, endTime, cubeInstance.getRootFactTable(), cubeInstance.getProject()); //load into kafka streamDataLoader.loadIntoKafka(data); logger.info("Write {} messages into {}", data.size(), streamDataLoader.toString()); //csv data for H2 use TableRef factTable = cubeInstance.getModel().getRootFactTable(); List<TblColRef> tableColumns = Lists.newArrayList(factTable.getColumns()); TimedJsonStreamParser timedJsonStreamParser = new TimedJsonStreamParser(tableColumns, null); StringBuilder sb = new StringBuilder(); for (String json : data) { List<String> rowColumns = timedJsonStreamParser .parse(ByteBuffer.wrap(json.getBytes(StandardCharsets.UTF_8))).get(0).getData(); sb.append(StringUtils.join(rowColumns, ",")); sb.append(System.getProperty("line.separator")); } appendFactTableData(sb.toString(), cubeInstance.getRootFactTable()); }
public boolean belongToFactTableDims(TblColRef tblColRef) { if (!belongToContextTables(tblColRef)) { return false; } KylinConfig kylinConfig = olapSchema.getConfig(); String projectName = olapSchema.getProjectName(); String factTableName = firstTableScan.getOlapTable().getTableName(); Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName, factTableName); for (IRealization real : realizations) { DataModelDesc model = real.getModel(); TblColRef.fixUnknownModel(model, tblColRef.getTableRef().getTableIdentity(), tblColRef); // cannot be a measure column Set<String> metrics = Sets.newHashSet(model.getMetrics()); if (metrics.contains(tblColRef.getIdentity())) { tblColRef.unfixTableRef(); return false; } // must belong to a fact table for (TableRef factTable : model.getFactTables()) { if (factTable.getColumns().contains(tblColRef)) { tblColRef.unfixTableRef(); return true; } } tblColRef.unfixTableRef(); } return false; }
List<TableExtDesc.ColumnStats> columnStatses = tableExtDesc.getColumnStats(); if (!columnStatses.isEmpty()) { for (TblColRef colRef : tableRef.getColumns()) { long cardinality = columnStatses.get(colRef.getColumnDesc().getZeroBasedIndex()) .getCardinality();
|| cube.getSnapshots().containsKey(digest.factTable)) { Set<TblColRef> dimCols = Sets.newHashSet(cube.getModel().findFirstTable(digest.factTable).getColumns());
public boolean belongToFactTableDims(TblColRef tblColRef) { if (!belongToContextTables(tblColRef)) { return false; } KylinConfig kylinConfig = olapSchema.getConfig(); String projectName = olapSchema.getProjectName(); String factTableName = firstTableScan.getOlapTable().getTableName(); Set<IRealization> realizations = ProjectManager.getInstance(kylinConfig).getRealizationsByTable(projectName, factTableName); for (IRealization real : realizations) { DataModelDesc model = real.getModel(); TblColRef.fixUnknownModel(model, tblColRef.getTableRef().getTableIdentity(), tblColRef); // cannot be a measure column Set<String> metrics = Sets.newHashSet(model.getMetrics()); if (metrics.contains(tblColRef.getIdentity())) { tblColRef.unfixTableRef(); return false; } // must belong to a fact table for (TableRef factTable : model.getFactTables()) { if (factTable.getColumns().contains(tblColRef)) { tblColRef.unfixTableRef(); return true; } } tblColRef.unfixTableRef(); } return false; }