@Override public void duplicate(String fromNamespace, String fromName, String toNamespace, String toName, boolean doOverwrite) throws DDFException { IPersistible from = this.load(fromNamespace, fromName); if (from instanceof DDF) { DDF to = (DDF) from; // to.setNamespace(toNamespace); to.getManager().setDDFName(to, toName); to.persist(); } else { throw new DDFException("Can only duplicate DDFs"); } }
private DDF createDDFWrapper() throws DDFException { DDF ddf = this.newContainerDDFImpl(); if (ddf == null) throw new DDFException(String.format("Cannot create new container DDF for %s: %s/%s", this.getClass(), this.getNamespace(), this.getName())); // Make sure we have a namespace and name if (Strings.isNullOrEmpty(this.getName())) this.setName(ddf.getSchemaHandler().newTableName(this)); // Make sure the DDF's names match ours ddf.getManager().setDDFName(ddf, this.getName()); return ddf; }
@Test(expected = DDFException.class) public void testRenamingDDF() throws DDFException { DDF ddf = this.getTestDDF(); DDFManager manager = this.getDDFManager(); manager.setDDFName(ddf, "myddf1"); String name1 = ddf.getName(); manager.setDDFName(ddf, "myddf2"); manager.getDDFByName(name1); }
@Test public void testMutableDDFBug() throws DDFException { ddf.setMutable(true); manager.setDDFName(ddf, "ddf_testMutableDDFBug"); ddf.Transform.transformUDF("col1 = (arrtime - deptime)"); ddf.Transform.transformUDF("col2 = (arrtime - arrdelay)"); } }
@Test public void testDDFMAnager() throws DDFException { DDF ddf = this.getTestDDF(); DDFManager manager = this.getDDFManager(); manager.setDDFName(ddf, "myddf"); UUID newUUID = UUID.randomUUID(); manager.setDDFUUID(ddf, newUUID); DDF ddf1 = manager.getDDFByName(ddf.getName()); Assert.assertEquals(ddf1.getUUID(), ddf.getUUID()); Assert.assertEquals(ddf1.getUUID(), newUUID); }
/** * Initialization to be done after constructor assignments, such as setting of the all-important DDFManager. */ protected void initialize(DDFManager manager, Object data, Class<?>[] typeSpecs, String name, Schema schema) throws DDFException { this.validateSchema(schema); this.setManager(manager); // this must be done first in case later stuff needs a manager if (typeSpecs != null) { this.getRepresentationHandler().set(data, typeSpecs); } this.getSchemaHandler().setSchema(schema); if(schema!= null && schema.getTableName() == null) { String tableName = this.getSchemaHandler().newTableName(); schema.setTableName(tableName); } manager.setDDFUUID(this, UUID.randomUUID()); if(!Strings.isNullOrEmpty(name)) manager.setDDFName(this, name); // Facades this.ML = new MLFacade(this, this.getMLSupporter()); this.VIEWS = new ViewsFacade(this, this.getViewHandler()); this.Transform = new TransformFacade(this, this.getTransformationHandler()); this.R = new RFacade(this, this.getAggregationHandler()); }
@Test public void testSimpleSparkDDFManager() throws DDFException { createTableAirline(); List<String> l = manager.sql("select * from airline", false).getRows(); Assert.assertEquals(31, l.size()); List<String> v = manager.sql("select count(*) from airline", false).getRows(); Assert.assertEquals(1, v.size()); Assert.assertEquals("31", v.get(0)); DDF ddf = manager.sql2ddf("select year, month, dayofweek, deptime, arrtime,origin, distance, arrdelay, " + "depdelay, carrierdelay, weatherdelay, nasdelay, securitydelay, lateaircraftdelay from airline", false); Assert.assertEquals(14, ddf.getSummary().length); manager.setDDFName(ddf, "myddf"); manager.addDDF(ddf); Assert.assertEquals(ddf, manager.getDDF(ddf.getUUID())); } }
/** * @brief Test ordinary spark query. * @throws DDFException */ @Test public void testLoading() throws DDFException { SQLDataSourceDescriptor sqlDataSourceDescriptor = new SQLDataSourceDescriptor(null, false, null, null, null); manager.sql("drop table if exists airline", sqlDataSourceDescriptor); manager.sql("create table airline (Year int,Month int,DayofMonth int," + "DayOfWeek int,DepTime int,CRSDepTime int,ArrTime int," + "CRSArrTime int,UniqueCarrier string, FlightNum int, " + "TailNum string, ActualElapsedTime int, CRSElapsedTime int, " + "AirTime int, ArrDelay int, DepDelay int, Origin string, " + "Dest string, Distance int, TaxiIn int, TaxiOut int, Cancelled int, " + "CancellationCode string, Diverted string, CarrierDelay int, " + "WeatherDelay int, NASDelay int, SecurityDelay int, LateAircraftDelay int ) " + "ROW FORMAT DELIMITED FIELDS TERMINATED BY ','", sqlDataSourceDescriptor); manager.sql("load data local inpath '../resources/test/airline.csv' " + "into table airline", sqlDataSourceDescriptor); DDF ddf = manager.sql2ddf("select year, month, dayofweek, deptime, arrtime,origin, distance, arrdelay, " + "depdelay, carrierdelay, weatherdelay, nasdelay, securitydelay, lateaircraftdelay from airline", sqlDataSourceDescriptor); this.manager.setDDFName(ddf, "airlineDDF"); // DDF sql2ddfRet = manager.sql2ddf("select * from " + // "ddf://adatao/airlineDDF"); }