/** * @brief Get the column information of this table. * @param name The URI or the name of the ddf. * @return The column information. * @throws DDFException */ private SqlResult describeTable(String name) throws DDFException { DDF ddf = this.getManager().getDDFByName(name); int colSize = ddf.getNumColumns(); List<String> ret = new ArrayList<String>(); for (int colIdx = 0; colIdx < colSize; ++colIdx) { Schema.Column col = ddf.getColumn(ddf.getColumnName(colIdx)); ret.add(col.getName().concat("\t").concat(col.getType().toString() .toLowerCase())); } List<Column> columnList = new ArrayList<Column>(); columnList.add(new Column("column_name", Schema.ColumnType.STRING)); columnList.add(new Column("value_type", Schema.ColumnType.STRING)); Schema schema = new Schema("table_info", columnList); return new SqlResult(schema, ret); }
public static void main(String[] args) throws DDFException { DDFManager manager = DDFManager.get(DDFManager.EngineType.SPARK); manager.sql("drop table if exists airline", false); manager.sql("create table airline (Year int,Month int,DayofMonth int," + "DayOfWeek int,DepTime int,CRSDepTime int,ArrTime int," + "CRSArrTime int,UniqueCarrier string, FlightNum int, " + "TailNum string, ActualElapsedTime int, CRSElapsedTime int, " + "AirTime int, ArrDelay int, DepDelay int, Origin string, " + "Dest string, Distance int, TaxiIn int, TaxiOut int, Cancelled int, " + "CancellationCode string, Diverted string, CarrierDelay int, " + "WeatherDelay int, NASDelay int, SecurityDelay int, LateAircraftDelay int ) " + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ','", false); manager.sql("load data local inpath 'resources/test/airline.csv' into table airline", false); DDF ddf = manager.sql2ddf("SELECT * FROM AIRLINE", false); long nrow = ddf.getNumRows(); int ncol = ddf.getNumColumns(); System.out.println("Number of data row is " + nrow); System.out.println("Number of data columns is " + ncol); } }
@Test public void testRemoveColumns() throws DDFException { createTableAirline(); DDF ddf = manager.sql2ddf("select * from airline", false); DDF ddf0 = manager.sql2ddf("select * from airline", false); List<String> columns = Lists.newArrayList(); columns.add("year"); columns.add("month"); columns.add("deptime"); ddf.VIEWS.removeColumn("year"); Assert.assertEquals(28, ddf.getNumColumns()); ddf.VIEWS.removeColumns("deptime"); Assert.assertEquals(27, ddf.getNumColumns()); ddf0.VIEWS.removeColumns(columns); Assert.assertEquals(26, ddf0.getNumColumns()); }
@Test public void testDropNA() throws DDFException { DDF newddfDropRow = ddf.dropNA(); DDF newddfDropColumn = ddf.dropNA(Axis.COLUMN); Assert.assertEquals(9, newddfDropRow.getNumRows()); Assert.assertEquals(22, newddfDropColumn.getNumColumns()); Assert.assertEquals(29, ddf.getMissingDataHandler().dropNA(Axis.COLUMN, NAChecking.ALL, 0, null).getNumColumns()); }
@Test public void testLoadFromJDBC() throws DDFException, URISyntaxException { // load data from a MySQL JDBCDataSourceDescriptor desc = new JDBCDataSourceDescriptor("jdbc:mysql://localhost:3306/test", "pauser", "papwd", "mtcars"); DDF ddf = manager.load(desc); Assert.assertTrue(ddf != null); Assert.assertEquals(ddf.getNumColumns(), 11); Assert.assertEquals(ddf.getNumRows(), 32); } }
@Test public void testSimpleAggregate() throws DDFException { // aggregation: select year, month, min(depdelay), max(arrdelay) from airline group by year, month; // Assert.assertEquals(13, ddf.aggregate("year, month, mean(depdelay), median(arrdelay)").size()); Assert.assertEquals(13, ddf.aggregate("year, month, avg(depdelay), stddev(arrdelay)").size()); Assert.assertEquals(2, ddf.aggregate("year, month, min(depdelay), max" + "(arrdelay)").get("2010\t3").length); Assert.assertEquals(0.87, ddf.correlation("arrdelay", "depdelay"), 0.5); // project subset Assert.assertEquals(3, ddf.VIEWS.project(new String[] { "year", "month", "deptime" }).getNumColumns()); Assert.assertEquals(5, ddf.VIEWS.head(5).size()); }
Assert.assertEquals(9, ddf.getNumColumns()); Assert.assertEquals("dist", ddf.getColumnName(8)); Assert.assertEquals(9, ddf.VIEWS.head(1).get(0).split("\\t").length); Assert.assertEquals(10, ddf.getNumColumns()); Assert.assertEquals(10, ddf.getSummary().length); Assert.assertEquals(5, ddf.getNumColumns()); Assert.assertEquals("speed", ddf.getColumnName(4)); ddf.setMutable(false); Assert.assertEquals(6, ddf3.getNumColumns()); Assert.assertEquals("speed", ddf3.getColumnName(5)); Assert.assertEquals(6, ddf3.getSummary().length); DDF ddf2 = ddf.Transform.transformUDF(s1, lcols); Assert.assertEquals(31, ddf2.getNumRows()); Assert.assertEquals(6, ddf2.getNumColumns());
DDF newddf = null; int numcols = this.getDDF().getNumColumns(); if (columns == null) { columns = this.getDDF().getColumnNames();
@Test public void testReservedFactor() throws DDFException { ddf.setAsFactor("year"); ddf.setAsFactor("month"); Assert.assertTrue(ddf.getSchema() != null); System.out.println(">>>>> column class = " + ddf.getColumn("year").getColumnClass()); System.out.println(">>>>> column class = " + ddf.getColumn("month").getColumnClass()); Assert.assertTrue(ddf.getColumn("year").getColumnClass() == Schema.ColumnClass.FACTOR); Assert.assertTrue(ddf.getColumn("month").getColumnClass() == Schema.ColumnClass.FACTOR); ddf.setMutable(true); ddf = ddf.Transform.transformUDF("test123= round(distance/2, 2)"); Assert.assertEquals(31, ddf.getNumRows()); Assert.assertEquals(9, ddf.getNumColumns()); Assert.assertEquals("test123", ddf.getColumnName(8)); Assert.assertEquals(9, ddf.VIEWS.head(1).get(0).split("\\t").length); System.out.println(">>>>> column class = " + ddf.getColumn("year").getColumnClass()); System.out.println(">>>>> column class = " + ddf.getColumn("month").getColumnClass()); Assert.assertTrue(ddf.getColumn("year").getColumnClass() == Schema.ColumnClass.FACTOR); Assert.assertTrue(ddf.getColumn("month").getColumnClass() == Schema.ColumnClass.FACTOR); Assert.assertTrue(ddf.getColumn("year").getOptionalFactor().getLevels().size() > 0); Assert.assertTrue(ddf.getColumn("month").getOptionalFactor().getLevels().size() > 0); System.out.println(">>>>>>>>>>>>> " + ddf.getSchema().getColumns()); }