public static synchronized HiveTestDataGenerator getInstance() throws Exception { if (instance == null) { final String dbDir = getTempDir("metastore_db"); final String whDir = getTempDir("warehouse"); instance = new HiveTestDataGenerator(dbDir, whDir); instance.generateTestData(); } return instance; }
private void testMultipleBatchJoin(final long right, final long left, final String joinType, final long expected) throws Exception { final String leftSide = BaseTestQuery.getTempDir("merge-join-left.json"); final String rightSide = BaseTestQuery.getTempDir("merge-join-right.json"); final BufferedWriter leftWriter = new BufferedWriter(new FileWriter(new File(leftSide))); final BufferedWriter rightWriter = new BufferedWriter(new FileWriter(new File(rightSide))); generateData(leftWriter, rightWriter, left, right); final String query1 = String.format("select count(*) c1 from dfs.\"%s\" L %s join dfs.\"%s\" R on L.k=R.k1", leftSide, joinType, rightSide); testBuilder() .sqlQuery(query1) .optionSettingQueriesForTestQuery("alter session set \"planner.enable_hashjoin\" = false") .unOrdered() .baselineColumns("c1") .baselineValues(expected) .go(); }
@Test public void testDrill4196() throws Exception { final String leftSide = BaseTestQuery.getTempDir("merge-join-left.json"); final String rightSide = BaseTestQuery.getTempDir("merge-join-right.json"); final BufferedWriter leftWriter = new BufferedWriter(new FileWriter(new File(leftSide))); final BufferedWriter rightWriter = new BufferedWriter(new FileWriter(new File(rightSide))); // output batch is 32k, create 60k left batch leftWriter.write(String.format("{ \"k\" : %d , \"v\": %d }", 9999, 9999)); for (int i=0; i < 6000; ++i) { leftWriter.write(String.format("{ \"k\" : %d , \"v\": %d }", 10000, 10000)); } leftWriter.write(String.format("{ \"k\" : %d , \"v\": %d }", 10001, 10001)); leftWriter.write(String.format("{ \"k\" : %d , \"v\": %d }", 10002, 10002)); // Keep all values same. Jon will consume entire right side. for (int i=0; i < 800; ++i) { rightWriter.write(String.format("{ \"k1\" : %d , \"v1\": %d }", 10000, 10000)); } leftWriter.close(); rightWriter.close(); final String query1 = String.format("select count(*) c1 from dfs.\"%s\" L %s join dfs.\"%s\" R on L.k=R.k1", leftSide, "inner", rightSide); testBuilder() .sqlQuery(query1) .optionSettingQueriesForTestQuery("alter session set \"planner.enable_hashjoin\" = false") .unOrdered() .baselineColumns("c1") .baselineValues(6000*800L) .go(); }
private HiveConf newHiveConf() { HiveConf conf = new HiveConf(SessionState.class); conf.set(ConfVars.METASTORECONNECTURLKEY.varname, String.format("jdbc:derby:;databaseName=%s;create=true", dbDir)); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); conf.set(ConfVars.METASTOREWAREHOUSE.varname, whDir); conf.set("mapred.job.tracker", "local"); conf.set(ConfVars.SCRATCHDIR.varname, getTempDir("scratch_dir")); conf.set(ConfVars.LOCALSCRATCHDIR.varname, getTempDir("local_scratch_dir")); conf.set(ConfVars.DYNAMICPARTITIONINGMODE.varname, "nonstrict"); conf.set(ConfVars.METASTORE_SCHEMA_VERIFICATION.varname, "false"); conf.set(ConfVars.METASTORE_AUTO_CREATE_ALL.varname, "true"); conf.set(ConfVars.HIVE_CBO_ENABLED.varname, "false"); return conf; }
final File regionDir = new File(BaseTestQuery.getTempDir("region")); regionDir.mkdirs(); final URL url = Resources.getResource("region.parquet"); getTempDir("part1"))); executeQuery(hiveDriver, String.format("INSERT INTO TABLE partition_pruning_test PARTITION(c=99, d=98, e=97) " +