private static void resetClientAndBit() throws Exception{ closeClient(); openClient(); }
public static synchronized HiveTestDataGenerator getInstance() throws Exception { if (instance == null) { final String dbDir = getTempDir("metastore_db"); final String whDir = getTempDir("warehouse"); instance = new HiveTestDataGenerator(dbDir, whDir); instance.generateTestData(); } return instance; }
public static void validateResults(final String query, final String tag) throws Exception { disableGlobalDictionary(); final FileStatus original = testAndGetResult(query, tag); enableGlobalDictionary(); FileStatus withDict = testAndGetResult(query, tag); // read file final boolean diff = compareFiles(original, withDict); if (!diff) { fail(format("Results do not match original data: [%s], with global dictionary: [%s]", original.getPath(), withDict.getPath())); } else { localFs.delete(original.getPath().getParent(), true); localFs.delete(withDict.getPath().getParent(), true); } }
public static FileStatus testAndGetResult(final String query, final String testName) throws Exception { final String tableName = format("%s%d", testName, random.nextInt(100000)); final String ctasQuery = format("CREATE TABLE dfs_test.%s STORE AS (type => 'json', prettyPrint => false) WITH SINGLE WRITER AS %s", tableName, query); final Path tableDir = new Path(getDfsTestTmpSchemaLocation(), tableName); test(ctasQuery); // find file final FileStatus fileStatus = localFs.listStatus(tableDir, new GlobFilter("*.json"))[0]; return fileStatus; }
public static void validateResultsOutOfOrder(final String query, final String tag) throws Exception { disableGlobalDictionary(); final FileStatus original = testAndGetResult(query, tag); enableGlobalDictionary(); FileStatus withDict = testAndGetResult(query, tag); boolean diff = false; final HashMap<String, Void> lines = new HashMap<>(); try (FSDataInputStream in1 = localFs.open(original.getPath())) { String line = null; while ((line = in1.readLine()) != null) { lines.put(line, null); } } try (FSDataInputStream in2 = localFs.open(withDict.getPath())) { String line = null; while ((line = in2.readLine()) != null) { if(!lines.containsKey(line)) { diff = true; break; } } } if (diff) { fail(format("Results do not match original data: [%s], with global dictionary: [%s]", original.getPath(), withDict.getPath())); } else { localFs.delete(original.getPath().getParent(), true); localFs.delete(withDict.getPath().getParent(), true); } } }
"OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' " + "TBLPROPERTIES ('avro.schema.url'='file:///%s')", BaseTestQuery.getPhysicalFileFromResource("avro_test_schema.json").replace('\\', '/')); final File regionDir = new File(BaseTestQuery.getTempDir("region")); regionDir.mkdirs(); final URL url = Resources.getResource("region.parquet"); getTempDir("part1"))); executeQuery(hiveDriver, String.format("INSERT INTO TABLE partition_pruning_test PARTITION(c=99, d=98, e=97) " +
@BeforeClass public static void setupDefaultTestCluster() throws Exception { // Create a test implementation of UserService // and inject it in SabotNode. BINDER_RULE.bind(UserService.class, new UserServiceTestImpl()); BaseTestQuery.setupDefaultTestCluster(); }
@AfterClass public static void closeClient() throws Exception { service.close(); BaseTestQuery.closeClient(); }
@Test public void testFilterSelectivityOptionsAbove1() throws Exception { String query3 = String.format("alter session set \"planner.filter.max_selectivity_estimate_factor\" = %f", 1.5); String errMsg3 = "Option planner.filter.max_selectivity_estimate_factor must be between 0.000000 and 1.000000"; BaseTestQuery.errorMsgTestHelper(query3, errMsg3); }
protected String getResultString(List<QueryDataBatch> results, String delimiter, boolean includeHeader) throws SchemaChangeException { final StringBuilder formattedResults = new StringBuilder(); final RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); for(final QueryDataBatch result : results) { loader.load(result.getHeader().getDef(), result.getData()); if (loader.getRecordCount() <= 0) { continue; } VectorUtil.appendVectorAccessibleContent(loader, formattedResults, delimiter, includeHeader); if (!includeHeader) { includeHeader = false; } loader.clear(); result.release(); } return formattedResults.toString(); }
/** * Utility method which tests given query produces a {@link UserException} with the expected * errorType. * @param testSqlQuery Test query * @param expectedErrorType Expected error type */ protected static void errorTypeTestHelper(final String testSqlQuery, final ErrorType expectedErrorType) { errorMsgWithTypeTestHelper(testSqlQuery, expectedErrorType, ""); }
@Test @Ignore("interval") public void checkReadWriteExtended() throws Exception { final String originalFile = "${WORKING_PATH}/src/test/resources/vector/complex/extended.json".replaceAll( Pattern.quote("${WORKING_PATH}"), Matcher.quoteReplacement(TestTools.getWorkingPath())); final String newTable = "TestExtendedTypes/newjson"; try { testNoResult(String.format("ALTER SESSION SET \"%s\" = 'json'", ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName())); testNoResult(String.format("ALTER SESSION SET \"%s\" = true", ExecConstants.JSON_EXTENDED_TYPES.getOptionName())); // create table test("create table dfs_test.tmp.\"%s\" as select * from dfs.\"%s\"", newTable, originalFile); // check query of table. test("select * from dfs_test.tmp.\"%s\"", newTable); // check that original file and new file match. final byte[] originalData = Files.readAllBytes(Paths.get(originalFile)); final byte[] newData = Files.readAllBytes(Paths.get(BaseTestQuery.getDfsTestTmpSchemaLocation() + '/' + newTable + "/0_0_0.json")); assertEquals(new String(originalData), new String(newData)); } finally { testNoResult(String.format("ALTER SESSION SET \"%s\" = '%s'", ExecConstants.OUTPUT_FORMAT_VALIDATOR.getOptionName(), ExecConstants.OUTPUT_FORMAT_VALIDATOR.getDefault().getValue())); testNoResult(String.format("ALTER SESSION SET \"%s\" = %s", ExecConstants.JSON_EXTENDED_TYPES.getOptionName(), ExecConstants.JSON_EXTENDED_TYPES.getDefault().getValue())); } }
@BeforeClass public static final void setupDefaultTestCluster() throws Exception { BaseTestQuery.setupDefaultTestCluster(); Map<String, FormatPluginConfig> formats = ImmutableMap.of("parquet", (FormatPluginConfig) new ParquetFormatConfig()); WorkspaceConfig workspace = new WorkspaceConfig(TEMPORARY_FOLDER.newFolder().getAbsolutePath(), true, "parquet"); String path = TEMPORARY_FOLDER.newFolder().getAbsolutePath(); BindingProvider p = getBindingProvider(); service = new PDFSService(p.provider(SabotContext.class), p.provider(FabricService.class), DremioTest.DEFAULT_SABOT_CONFIG, getSabotContext().getAllocator()); service.start(); SourceConfig c = new SourceConfig(); PDFSConf conf = new PDFSConf() ; conf.path = path; c.setType(conf.getType()); c.setName("pdfs"); c.setMetadataPolicy(CatalogService.DEFAULT_METADATA_POLICY); c.setConfig(conf.toBytesString()); ((CatalogServiceImpl) getSabotContext().getCatalogService()).getSystemUserCatalog().createSource(c); }
@Test public void testFilterSelectivityOptionsNegative() throws Exception { String query1 = String.format("alter session set \"planner.filter.min_selectivity_estimate_factor\" = %f", -0.5); String errMsg1 = "Option planner.filter.min_selectivity_estimate_factor must be between 0.000000 and 1.000000"; BaseTestQuery.errorMsgTestHelper(query1, errMsg1); }
protected int printResult(List<QueryDataBatch> results) throws SchemaChangeException { int rowCount = 0; final RecordBatchLoader loader = new RecordBatchLoader(getAllocator()); for(final QueryDataBatch result : results) { rowCount += result.getHeader().getRowCount(); loader.load(result.getHeader().getDef(), result.getData()); // TODO: Clean: DRILL-2933: That load(...) no longer throws // SchemaChangeException, so check/clean throw clause above. VectorUtil.showVectorAccessibleContent(loader, columnWidths); loader.clear(); result.release(); } System.out.println("Total record count: " + rowCount); return rowCount; }
protected static void updateTestCluster(int newNodeCount, SabotConfig newConfig) { Preconditions.checkArgument(newNodeCount > 0, "Number of Nodes must be at least one"); if (nodeCount != newNodeCount || config != null) { // TODO: Currently we have to shutdown the existing SabotNode cluster before starting a new one with the given // SabotNode count. Revisit later to avoid stopping the cluster. try { closeClient(); nodeCount = newNodeCount; if (newConfig != null) { // For next test class, updated SabotConfig will be replaced by default SabotConfig in BaseTestQuery as part // of the @BeforeClass method of test class. config = newConfig; } openClient(); } catch(Exception e) { throw new RuntimeException("Failure while updating the test SabotNode cluster.", e); } } }
private void testMultipleBatchJoin(final long right, final long left, final String joinType, final long expected) throws Exception { final String leftSide = BaseTestQuery.getTempDir("merge-join-left.json"); final String rightSide = BaseTestQuery.getTempDir("merge-join-right.json"); final BufferedWriter leftWriter = new BufferedWriter(new FileWriter(new File(leftSide))); final BufferedWriter rightWriter = new BufferedWriter(new FileWriter(new File(rightSide))); generateData(leftWriter, rightWriter, left, right); final String query1 = String.format("select count(*) c1 from dfs.\"%s\" L %s join dfs.\"%s\" R on L.k=R.k1", leftSide, joinType, rightSide); testBuilder() .sqlQuery(query1) .optionSettingQueriesForTestQuery("alter session set \"planner.enable_hashjoin\" = false") .unOrdered() .baselineColumns("c1") .baselineValues(expected) .go(); }
@Test public void testDrill4196() throws Exception { final String leftSide = BaseTestQuery.getTempDir("merge-join-left.json"); final String rightSide = BaseTestQuery.getTempDir("merge-join-right.json"); final BufferedWriter leftWriter = new BufferedWriter(new FileWriter(new File(leftSide))); final BufferedWriter rightWriter = new BufferedWriter(new FileWriter(new File(rightSide))); // output batch is 32k, create 60k left batch leftWriter.write(String.format("{ \"k\" : %d , \"v\": %d }", 9999, 9999)); for (int i=0; i < 6000; ++i) { leftWriter.write(String.format("{ \"k\" : %d , \"v\": %d }", 10000, 10000)); } leftWriter.write(String.format("{ \"k\" : %d , \"v\": %d }", 10001, 10001)); leftWriter.write(String.format("{ \"k\" : %d , \"v\": %d }", 10002, 10002)); // Keep all values same. Jon will consume entire right side. for (int i=0; i < 800; ++i) { rightWriter.write(String.format("{ \"k1\" : %d , \"v1\": %d }", 10000, 10000)); } leftWriter.close(); rightWriter.close(); final String query1 = String.format("select count(*) c1 from dfs.\"%s\" L %s join dfs.\"%s\" R on L.k=R.k1", leftSide, "inner", rightSide); testBuilder() .sqlQuery(query1) .optionSettingQueriesForTestQuery("alter session set \"planner.enable_hashjoin\" = false") .unOrdered() .baselineColumns("c1") .baselineValues(6000*800L) .go(); }
private HiveConf newHiveConf() { HiveConf conf = new HiveConf(SessionState.class); conf.set(ConfVars.METASTORECONNECTURLKEY.varname, String.format("jdbc:derby:;databaseName=%s;create=true", dbDir)); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); conf.set(ConfVars.METASTOREWAREHOUSE.varname, whDir); conf.set("mapred.job.tracker", "local"); conf.set(ConfVars.SCRATCHDIR.varname, getTempDir("scratch_dir")); conf.set(ConfVars.LOCALSCRATCHDIR.varname, getTempDir("local_scratch_dir")); conf.set(ConfVars.DYNAMICPARTITIONINGMODE.varname, "nonstrict"); conf.set(ConfVars.METASTORE_SCHEMA_VERIFICATION.varname, "false"); conf.set(ConfVars.METASTORE_AUTO_CREATE_ALL.varname, "true"); conf.set(ConfVars.HIVE_CBO_ENABLED.varname, "false"); return conf; }