/** * check whether all columns in `table` are still in `fields` and have the same index as before. * * @param table kylin's table metadata * @param fields current table metadata in hive * @return true if only new columns are appended in hive, false otherwise */ private boolean checkAllColumnsInTableDesc(TableDesc origTable, TableDesc newTable) { if (origTable.getColumnCount() > newTable.getColumnCount()) { return false; } ColumnDesc[] columns = origTable.getColumns(); for (int i = 0; i < columns.length; i++) { if (!isColumnCompatible(columns[i], newTable.getColumns()[i])) { return false; } } return true; }
@Test public void testLoadTableMetadata() throws SQLException { String tableName = "tb1"; String databaseName = "testdb"; ResultSet rs1 = mock(ResultSet.class); when(rs1.next()).thenReturn(true).thenReturn(false); when(rs1.getString("TABLE_TYPE")).thenReturn("TABLE"); ResultSet rs2 = mock(ResultSet.class); when(rs2.next()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false); when(rs2.getString("COLUMN_NAME")).thenReturn("COL1").thenReturn("COL2").thenReturn("COL3"); when(rs2.getInt("DATA_TYPE")).thenReturn(Types.VARCHAR).thenReturn(Types.INTEGER).thenReturn(Types.DECIMAL); when(rs2.getInt("COLUMN_SIZE")).thenReturn(128).thenReturn(10).thenReturn(19); when(rs2.getInt("DECIMAL_DIGITS")).thenReturn(0).thenReturn(0).thenReturn(4); when(rs2.getInt("ORDINAL_POSITION")).thenReturn(1).thenReturn(3).thenReturn(2); when(rs2.getString("REMARKS")).thenReturn("comment1").thenReturn("comment2").thenReturn("comment3"); when(jdbcMetadata.getTable(dbmd, databaseName, tableName)).thenReturn(rs1); when(jdbcMetadata.listColumns(dbmd, databaseName, tableName)).thenReturn(rs2); Pair<TableDesc, TableExtDesc> result = jdbcExplorer.loadTableMetadata(databaseName, tableName, "proj"); TableDesc tableDesc = result.getFirst(); ColumnDesc columnDesc = tableDesc.getColumns()[1]; Assert.assertEquals(databaseName.toUpperCase(Locale.ROOT), tableDesc.getDatabase()); Assert.assertEquals(3, tableDesc.getColumnCount()); Assert.assertEquals("TABLE", tableDesc.getTableType()); Assert.assertEquals("COL2", columnDesc.getName()); Assert.assertEquals("integer", columnDesc.getTypeName()); Assert.assertEquals("comment2", columnDesc.getComment()); Assert.assertEquals(databaseName.toUpperCase(Locale.ROOT) + "." + tableName.toUpperCase(Locale.ROOT), result.getSecond().getIdentity()); }
while (reader.next()) { String[] row = reader.getRow(); int[] rowIndex = new int[tableDesc.getColumnCount()]; for (ColumnDesc column : tableDesc.getColumns()) { rowIndex[column.getZeroBasedIndex()] = dict.getIdFromValue(row[column.getZeroBasedIndex()]);
@Test public void testBasics() throws Exception { TableMetadataManager tblManager = TableMetadataManager.getInstance(getTestConfig()); TableDesc tblDesc = tblManager.getTableDesc("test_kylin_fact", "default"); IReadableTable table = SourceManager.getSource(new JdbcSourceTest.JdbcSourceAware()) .createReadableTable(tblDesc, null); // test TableReader try (IReadableTable.TableReader reader = table.getReader()) { Assert.assertTrue(reader instanceof JdbcTableReader); Assert.assertTrue(table instanceof JdbcTable); Assert.assertTrue(reader.next()); String[] row = reader.getRow(); Assert.assertNotNull(row); Assert.assertEquals(tblDesc.getColumnCount(), row.length); } // test basics Assert.assertTrue(table.exists()); IReadableTable.TableSignature sign = table.getSignature(); Assert.assertNotNull(sign); Assert.assertEquals(String.format(Locale.ROOT, "%s.%s", tblDesc.getDatabase(), tblDesc.getName()), sign.getPath()); Assert.assertTrue(sign.getLastModifiedTime() > 0); }
public HiveTable(MetadataManager metaMgr, String table) { TableDesc tableDesc = metaMgr.getTableDesc(table); this.database = tableDesc.getDatabase(); this.hiveTable = tableDesc.getName(); this.nColumns = tableDesc.getColumnCount(); }
@Test public void testLoadTableMetadata() throws Exception { Pair<TableDesc, TableExtDesc> pair = explorer.loadTableMetadata("DEFAULT", "TEST_KYLIN_FACT", "DEFAULT"); Assert.assertNotNull(pair.getFirst()); Assert.assertNotNull(pair.getSecond()); TableDesc tblDesc = pair.getFirst(); TableExtDesc tblExtDesc = pair.getSecond(); Assert.assertEquals("TEST_KYLIN_FACT", tblDesc.getName()); Assert.assertEquals("TABLE", tblDesc.getTableType()); Assert.assertEquals("DEFAULT.TEST_KYLIN_FACT", tblDesc.getIdentity()); Assert.assertEquals("DEFAULT", tblDesc.getDatabase()); Assert.assertEquals("DEFAULT", tblDesc.getProject()); Assert.assertEquals(tblDesc.getIdentity(), tblExtDesc.getIdentity()); Assert.assertEquals(tblDesc.getProject(), tblExtDesc.getProject()); ColumnDesc[] columnDescs = tblDesc.getColumns(); Assert.assertEquals(tblDesc.getColumnCount(), columnDescs.length); Assert.assertNotNull(columnDescs[0].getName()); Assert.assertNotNull(columnDescs[0].getDatatype()); Assert.assertNotNull(columnDescs[0].getType()); Assert.assertNotNull(columnDescs[0].getId()); }
valueCols = new int[IIDimension.getColumnCount(valueDimensions)]; metricsCols = new int[metricNames.length]; metricsColSet = new BitSet(this.getTableDesc(this.getFactTableName()).getColumnCount());
srcCol = col.getName(); srcColIdx = col.getColumn().getZeroBasedIndex(); int nColumns = metaMgr.getTableDesc(col.getTable()).getColumnCount(); table = new FileTable(factColumnsPath + "/" + col.getName(), nColumns); return new Object[] { srcTable, srcCol, srcColIdx, table };
valueCols = new int[ModelDimensionDesc.getColumnCount(valueDimensions)]; metricsCols = new int[metricNames.length]; metricsColSet = new BitSet(this.getTableDesc(this.getFactTableName()).getColumnCount());
while (reader.next()) { String[] row = reader.getRow(); int[] rowIndex = new int[tableDesc.getColumnCount()]; for (ColumnDesc column : tableDesc.getColumns()) { rowIndex[column.getZeroBasedIndex()] = dict.getIdFromValue(row[column.getZeroBasedIndex()]);
srcCol = col.getName(); srcColIdx = col.getColumn().getZeroBasedIndex(); int nColumns = metaMgr.getTableDesc(col.getTable()).getColumnCount(); table = new FileTable(factColumnsPath + "/" + col.getName(), nColumns); return new Object[] { srcTable, srcCol, srcColIdx, table };