public static HashSet<String> extractDatabaseNames(List<TableDesc> tables) { HashSet<String> databaseNames = new HashSet<String>(); for (TableDesc tableDesc : tables) { databaseNames.add(tableDesc.getDatabase()); } return databaseNames; } }
public static HashMap<String, Integer> extractDatabaseOccurenceCounts(Collection<TableDesc> tables) { HashMap<String, Integer> databaseCounts = new HashMap<String, Integer>(); for (TableDesc tableDesc : tables) { String databaseName = tableDesc.getDatabase(); Integer counter = databaseCounts.get(databaseName); if (counter != null) databaseCounts.put(databaseName, counter + 1); else databaseCounts.put(databaseName, 1); } return databaseCounts; }
public String getIdentity() { if (identity == null) { identity = String.format(Locale.ROOT, "%s.%s", this.getDatabase().toUpperCase(Locale.ROOT), this.getName()) .toUpperCase(Locale.ROOT); } return identity; }
private void generateDatabaseDDL(Set<TableDesc> tables, PrintWriter out) { Set<String> dbs = new HashSet<>(); for (TableDesc t : tables) { String db = t.getDatabase(); if (StringUtils.isBlank(db) == false && "DEFAULT".equals(db) == false) dbs.add(db); } for (String db : dbs) { out.print("CREATE DATABASE IF NOT EXISTS " + normHiveIdentifier(db) + ";\n"); } out.print("\n"); }
public JdbcTable(JdbcConnector dataSource, TableDesc tableDesc) { this.dataSource = dataSource; this.database = tableDesc.getDatabase(); this.tableName = tableDesc.getName(); }
public String getIdentityQuoted(String quot) { String dbName = quot + this.getDatabase() + quot; String tableName = quot + this.getName() + quot; return String.format(Locale.ROOT, "%s.%s", dbName, tableName).toUpperCase(Locale.ROOT); }
public JdbcTable(TableDesc tableDesc) { this.database = tableDesc.getDatabase(); this.tableName = tableDesc.getName(); }
public void validateTable(String project, String table) throws IOException { List<TableDesc> tableDescs = tableService.getTableDescByProject(project, false); Set<String> tables = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (TableDesc tableDesc : tableDescs) { tables.add(tableDesc.getDatabase() + "." + tableDesc.getName()); } if (!tables.contains(table)) { throw new RuntimeException("Operation failed, table:" + table + " not exists"); } }
public void init(KylinConfig config, String project) { this.project = project; this.config = config; if (name != null) name = name.toUpperCase(Locale.ROOT); if (getDatabase() != null) setDatabase(getDatabase().toUpperCase(Locale.ROOT)); if (columns != null) { Arrays.parallelSort(columns, new Comparator<ColumnDesc>() { @Override public int compare(ColumnDesc col1, ColumnDesc col2) { Integer id1 = Integer.parseInt(col1.getId()); Integer id2 = Integer.parseInt(col2.getId()); return id1.compareTo(id2); } }); for (ColumnDesc col : columns) { col.init(this); } } }
public HiveTable(TableDesc tableDesc) { this.database = tableDesc.getDatabase(); this.hiveTable = tableDesc.getName(); try { this.hiveTableMeta = getHiveClient().getHiveTableMeta(database, hiveTable); } catch (Exception e) { throw new RuntimeException("cannot get HiveTableMeta", e); } }
private String[] generateCreateTableSql(TableDesc tableDesc) { logger.info("Generate create table sql: {}", tableDesc); String tableIdentity = String .format(Locale.ROOT, "%s.%s", tableDesc.getDatabase().toUpperCase(Locale.ROOT), tableDesc.getName()) .toUpperCase(Locale.ROOT); String dropsql = "DROP TABLE IF EXISTS " + tableIdentity; String dropsql2 = "DROP VIEW IF EXISTS " + tableIdentity; StringBuilder ddl = new StringBuilder(); ddl.append("CREATE TABLE " + tableIdentity + "\n"); ddl.append("(" + "\n"); for (int i = 0; i < tableDesc.getColumns().length; i++) { ColumnDesc col = tableDesc.getColumns()[i]; if (i > 0) { ddl.append(","); } ddl.append(col.getName() + " " + getSqlDataType((col.getDatatype())) + "\n"); } ddl.append(")"); return new String[] { dropsql, dropsql2, ddl.toString() }; }
private Map<String, Table> buildTableMap() { Map<String, Table> olapTables = new HashMap<String, Table>(); Collection<TableDesc> projectTables = ProjectManager.getInstance(config).listExposedTables(projectName, exposeMore); for (TableDesc tableDesc : projectTables) { if (tableDesc.getDatabase().equals(schemaName)) { final String tableName = tableDesc.getName();//safe to use tableDesc.getName() here, it is in a DB context now final OLAPTable table = new OLAPTable(this, tableDesc, exposeMore); olapTables.put(tableName, table); //logger.debug("Project " + projectName + " exposes table " + tableName); } } return olapTables; }
private Set<String> getAllColumns(String project, String table) throws IOException { List<TableDesc> tableDescByProject = tableService.getTableDescByProject(project, true); Set<String> cols = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (TableDesc tableDesc : tableDescByProject) { String tbl = tableDesc.getDatabase() + "." + tableDesc.getName(); if (tbl.equalsIgnoreCase(table)) { for (ColumnDesc column : tableDesc.getColumns()) { cols.add(column.getName()); } break; } } return cols; } }
public List<Pair<TableDesc, TableExtDesc>> extractHiveTableMeta(String[] tables, String project) throws Exception { // de-dup SetMultimap<String, String> db2tables = LinkedHashMultimap.create(); for (String fullTableName : tables) { String[] parts = HadoopUtil.parseHiveTableName(fullTableName); db2tables.put(parts[0], parts[1]); } // load all tables first List<Pair<TableDesc, TableExtDesc>> allMeta = Lists.newArrayList(); ProjectInstance projectInstance = getProjectManager().getProject(project); ISourceMetadataExplorer explr = SourceManager.getSource(projectInstance).getSourceMetadataExplorer(); for (Map.Entry<String, String> entry : db2tables.entries()) { Pair<TableDesc, TableExtDesc> pair = explr.loadTableMetadata(entry.getKey(), entry.getValue(), project); TableDesc tableDesc = pair.getFirst(); Preconditions.checkState(tableDesc.getDatabase().equals(entry.getKey().toUpperCase(Locale.ROOT))); Preconditions.checkState(tableDesc.getName().equals(entry.getValue().toUpperCase(Locale.ROOT))); Preconditions.checkState(tableDesc.getIdentity() .equals(entry.getKey().toUpperCase(Locale.ROOT) + "." + entry.getValue().toUpperCase(Locale.ROOT))); TableExtDesc extDesc = pair.getSecond(); Preconditions.checkState(tableDesc.getIdentity().equals(extDesc.getIdentity())); allMeta.add(pair); } return allMeta; }
public TableDesc(TableDesc other) { this.uuid = other.uuid; this.lastModified = other.lastModified; this.name = other.name; this.sourceType = other.sourceType; this.tableType = other.tableType; this.dataGen = other.dataGen; this.columns = new ColumnDesc[other.columns.length]; for (int i = 0; i < other.columns.length; i++) { this.columns[i] = new ColumnDesc(other.columns[i]); this.columns[i].init(this); } this.project = other.project; this.config = other.config; this.database.setName(other.getDatabase()); this.identity = other.identity; }
protected static String getTableNameForHCat(TableDesc table, String uuid) { String tableName = (table.isView()) ? table.getMaterializedName(uuid) : table.getName(); String database = (table.isView()) ? KylinConfig.getInstanceFromEnv().getHiveDatabaseForIntermediateTable() : table.getDatabase(); return String.format(Locale.ROOT, "%s.%s", database, tableName).toUpperCase(Locale.ROOT); }
/** * @param table */ public TableDescResponse(TableDesc table) { this.setColumns(table.getColumns()); this.setDatabase(table.getDatabase()); this.setName(table.getName()); this.setSourceType(table.getSourceType()); this.setUuid(table.getUuid()); this.setTableType(table.getTableType()); }
@Test public void testReadTableDesc() throws IOException { String requestTableData = "{\"name\":\"my_table_name\",\"source_type\":1,\"columns\":[{\"id\":1,\"name\":" + "\"amount\",\"datatype\":\"decimal\"},{\"id\":2,\"name\":\"category\",\"datatype\":\"varchar(256)\"}," + "{\"id\":3,\"name\":\"order_time\",\"datatype\":\"timestamp\"},{\"id\":4,\"name\":\"device\"," + "\"datatype\":\"varchar(256)\"},{\"id\":5,\"name\":\"qty\",\"datatype\":\"int\"},{\"id\":6,\"name\":" + "\"user_id\",\"datatype\":\"varchar(256)\"},{\"id\":7,\"name\":\"user_age\",\"datatype\":\"int\"}," + "{\"id\":8,\"name\":\"user_gender\",\"datatype\":\"varchar(256)\"},{\"id\":9,\"name\":\"currency\"," + "\"datatype\":\"varchar(256)\"},{\"id\":10,\"name\":\"country\",\"datatype\":\"varchar(256)\"}," + "{\"id\":11,\"name\":\"year_start\",\"datatype\":\"date\"},{\"id\":12,\"name\":\"quarter_start\"," + "\"datatype\":\"date\"},{\"id\":13,\"name\":\"month_start\",\"datatype\":\"date\"},{\"id\":14," + "\"name\":\"week_start\",\"datatype\":\"date\"},{\"id\":15,\"name\":\"day_start\",\"datatype\":" + "\"date\"},{\"id\":16,\"name\":\"hour_start\",\"datatype\":\"timestamp\"},{\"id\":17,\"name\":" + "\"minute_start\",\"datatype\":\"timestamp\"}],\"database\":\"my_database_name\"}"; TableDesc desc = JsonUtil.readValue(requestTableData, TableDesc.class); String[] dbTable = HadoopUtil.parseHiveTableName(desc.getIdentity()); desc.setName(dbTable[1]); desc.setDatabase(dbTable[0]); Assert.assertEquals("my_table_name".toUpperCase(Locale.ROOT), desc.getName()); Assert.assertEquals("my_database_name".toUpperCase(Locale.ROOT), desc.getDatabase()); } }
@Test public void testBasics() throws Exception { TableMetadataManager tblManager = TableMetadataManager.getInstance(getTestConfig()); TableDesc tblDesc = tblManager.getTableDesc("test_kylin_fact", "default"); IReadableTable table = SourceManager.getSource(new JdbcSourceTest.JdbcSourceAware()) .createReadableTable(tblDesc, null); // test TableReader try (IReadableTable.TableReader reader = table.getReader()) { Assert.assertTrue(reader instanceof JdbcTableReader); Assert.assertTrue(table instanceof JdbcTable); Assert.assertTrue(reader.next()); String[] row = reader.getRow(); Assert.assertNotNull(row); Assert.assertEquals(tblDesc.getColumnCount(), row.length); } // test basics Assert.assertTrue(table.exists()); IReadableTable.TableSignature sign = table.getSignature(); Assert.assertNotNull(sign); Assert.assertEquals(String.format(Locale.ROOT, "%s.%s", tblDesc.getDatabase(), tblDesc.getName()), sign.getPath()); Assert.assertTrue(sign.getLastModifiedTime() > 0); }
@Test public void testLoadTableMetadata() throws Exception { Pair<TableDesc, TableExtDesc> pair = explorer.loadTableMetadata("DEFAULT", "TEST_KYLIN_FACT", "DEFAULT"); Assert.assertNotNull(pair.getFirst()); Assert.assertNotNull(pair.getSecond()); TableDesc tblDesc = pair.getFirst(); TableExtDesc tblExtDesc = pair.getSecond(); Assert.assertEquals("TEST_KYLIN_FACT", tblDesc.getName()); Assert.assertEquals("TABLE", tblDesc.getTableType()); Assert.assertEquals("DEFAULT.TEST_KYLIN_FACT", tblDesc.getIdentity()); Assert.assertEquals("DEFAULT", tblDesc.getDatabase()); Assert.assertEquals("DEFAULT", tblDesc.getProject()); Assert.assertEquals(tblDesc.getIdentity(), tblExtDesc.getIdentity()); Assert.assertEquals(tblDesc.getProject(), tblExtDesc.getProject()); ColumnDesc[] columnDescs = tblDesc.getColumns(); Assert.assertEquals(tblDesc.getColumnCount(), columnDescs.length); Assert.assertNotNull(columnDescs[0].getName()); Assert.assertNotNull(columnDescs[0].getDatatype()); Assert.assertNotNull(columnDescs[0].getType()); Assert.assertNotNull(columnDescs[0].getId()); }