private SamzaSqlApplicationRunner(SamzaApplication app, Boolean localRunner, Config config) { this.runner = ApplicationRunners.getApplicationRunner(app, computeSamzaConfigs(localRunner, config)); }
public void runAndWaitForFinish(ExternalContext externalContext) { Validate.isTrue(runner instanceof LocalApplicationRunner, "This method can be called only in standalone mode."); run(externalContext); waitForFinish(); }
@Override public NonQueryResult executeNonQuery(ExecutionContext context, List<String> statement) { lastErrorMsg = ""; int execId = execIdSeq.incrementAndGet(); Map<String, String> staticConfigs = fetchSamzaSqlConfig(execId, context); staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(formatSqlStmts(statement))); SamzaSqlApplicationRunner runner; try { runner = new SamzaSqlApplicationRunner(true, new MapConfig(staticConfigs)); runner.run(null); } catch (SamzaException ex) { String msg = "Execution of the query failed with exception "; lastErrorMsg = msg + ex.toString(); LOG.error(msg, ex); return new NonQueryResult(execId, false); } executions.put(execId, runner); LOG.debug("Executing sql. Id ", execId); return new NonQueryResult(execId, true); }
@Override public boolean removeExecution(ExecutionContext context, int exeId) { lastErrorMsg = ""; SamzaSqlApplicationRunner runner = executions.get(exeId); if (runner != null) { if (runner.status().getStatusCode().equals(ApplicationStatus.StatusCode.Running)) { lastErrorMsg = "Trying to remove a ongoing execution " + exeId; LOG.error(lastErrorMsg); return false; } executions.remove(exeId); LOG.debug("Stopping execution ", exeId); return true; } else { lastErrorMsg = "Trying to remove a non-existing SQL execution " + exeId; LOG.warn(lastErrorMsg); return false; } }
public void runAndWaitForFinish() { runAndWaitForFinish(null); }
@Override public boolean stopExecution(ExecutionContext context, int exeId) { lastErrorMsg = ""; SamzaSqlApplicationRunner runner = executions.get(exeId); if (runner != null) { LOG.debug("Stopping execution ", exeId); try { runner.kill(); } catch (SamzaException ex) { String msg = "Stopping execution failed with exception "; lastErrorMsg = msg + ex.toString(); LOG.warn(msg, ex); return false; } try { Thread.sleep(500); // wait for a second } catch (InterruptedException e) { e.printStackTrace(); } return true; } else { lastErrorMsg = "Trying to stop a non-existing SQL execution " + exeId; LOG.warn(lastErrorMsg); return false; } }
private ExecutionStatus queryExecutionStatus(SamzaSqlApplicationRunner runner) { lastErrorMsg = ""; switch (runner.status().getStatusCode()) { case New: return ExecutionStatus.New; case Running: return ExecutionStatus.Running; case SuccessfulFinish: return ExecutionStatus.SuccessfulFinish; case UnsuccessfulFinish: return ExecutionStatus.UnsuccessfulFinish; default: lastErrorMsg = String.format("Unsupported execution status %s", runner.status().getStatusCode().toString()); return null; } }
private SamzaSqlApplicationRunner(SamzaApplication app, Boolean localRunner, Config config) { this.runner = ApplicationRunners.getApplicationRunner(app, computeSamzaConfigs(localRunner, config)); }
public void runAndWaitForFinish() { Validate.isTrue(runner instanceof LocalApplicationRunner, "This method can be called only in standalone mode."); run(); waitForFinish(); }
@Override public QueryResult executeQuery(ExecutionContext context, String statement) { lastErrorMsg = ""; outputData.clear(); int execId = execIdSeq.incrementAndGet(); Map<String, String> staticConfigs = fetchSamzaSqlConfig(execId, context); List<String> sqlStmts = formatSqlStmts(Collections.singletonList(statement)); staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts)); SamzaSqlApplicationRunner runner; try { runner = new SamzaSqlApplicationRunner(true, new MapConfig(staticConfigs)); runner.run(null); } catch (SamzaException ex) { String msg = "Execution failed with exception "; lastErrorMsg = msg + ex.toString(); LOG.error(msg, ex); return new QueryResult(execId, null, false); } executions.put(execId, runner); LOG.debug("Executing sql. Id ", execId); return new QueryResult(execId, generateResultSchema(new MapConfig(staticConfigs)), true); }
@Test public void testComputeSamzaConfigs() { Map<String, String> configs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10); String sql1 = "Insert into testavro.outputTopic(id,long_value) select id, MyTest(id) as long_value from testavro.SIMPLE1"; configs.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql1); configs.put(SamzaSqlApplicationRunner.RUNNER_CONFIG, SamzaSqlApplicationRunner.class.getName()); MapConfig samzaConfig = new MapConfig(configs); Config newConfigs = SamzaSqlApplicationRunner.computeSamzaConfigs(true, samzaConfig); Assert.assertEquals(newConfigs.get(SamzaSqlApplicationRunner.RUNNER_CONFIG), LocalApplicationRunner.class.getName()); // Check whether five new configs added. Assert.assertEquals(newConfigs.size(), configs.size() + 5); newConfigs = SamzaSqlApplicationRunner.computeSamzaConfigs(false, samzaConfig); Assert.assertEquals(newConfigs.get(SamzaSqlApplicationRunner.RUNNER_CONFIG), RemoteApplicationRunner.class.getName()); // Check whether five new configs added. Assert.assertEquals(newConfigs.size(), configs.size() + 5); } }
@Test (expected = SamzaException.class) public void testTranslateStreamTableCrossJoin() { Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1); String sql = "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)" + " select p.name as profileName, pv.pageKey" + " from testavro.PAGEVIEW as pv, testavro.PROFILE.`$table` as p"; config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql); Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config)); List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList())); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig); QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig); translator.translate(queryInfo.get(0), streamAppDesc, 0); }
@Test (expected = SamzaException.class) public void testTranslateStreamTableInnerJoinWithMissingStream() { Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 10); String configIOResolverDomain = String.format(SamzaSqlApplicationConfig.CFG_FMT_SOURCE_RESOLVER_DOMAIN, "config"); config.put(configIOResolverDomain + SamzaSqlApplicationConfig.CFG_FACTORY, ConfigBasedIOResolverFactory.class.getName()); String sql = "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)" + " select p.name as profileName, pv.pageKey" + " from testavro.PAGEVIEW as pv" + " join testavro.`$table` as p" + " on p.id = pv.profileId"; config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql); Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config)); List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList())); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig); QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig); translator.translate(queryInfo.get(0), streamAppDesc, 0); }
@Test (expected = SamzaException.class) public void testTranslateStreamTableJoinWithSubQuery() { Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1); String sql = "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)" + " select p.name as profileName, pv.pageKey" + " from testavro.PAGEVIEW as pv" + " where exists " + " (select p.id from testavro.PROFILE.`$table` as p" + " where p.id = pv.profileId)"; config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql); Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config)); List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList())); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig); QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig); translator.translate(queryInfo.get(0), streamAppDesc, 0); }
@Test (expected = SamzaException.class) public void testTranslateStreamTableJoinWithoutJoinOperator() { Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1); String sql = "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)" + " select p.name as profileName, pv.pageKey" + " from testavro.PAGEVIEW as pv, testavro.PROFILE.`$table` as p" + " where p.id = pv.profileId"; config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql); Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config)); List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList())); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig); QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig); translator.translate(queryInfo.get(0), streamAppDesc, 0); }
@Test (expected = SamzaException.class) public void testTranslateStreamTableJoinWithFullJoinOperator() { Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1); String sql = "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)" + " select p.name as profileName, pv.pageKey" + " from testavro.PAGEVIEW as pv" + " full join testavro.PROFILE.`$table` as p" + " on p.id = pv.profileId"; config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql); Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config)); List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList())); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig); QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig); translator.translate(queryInfo.get(0), streamAppDesc, 0); }
@Test (expected = SamzaException.class) public void testTranslateTableTableJoin() { Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1); String sql = "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)" + " select p.name as profileName, pv.pageKey" + " from testavro.PAGEVIEW.`$table` as pv" + " join testavro.PROFILE.`$table` as p" + " on p.id = pv.profileId"; config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql); Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config)); List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList())); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig); QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig); translator.translate(queryInfo.get(0), streamAppDesc, 0); }
@Test (expected = SamzaException.class) public void testTranslateStreamStreamJoin() { Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1); String sql = "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)" + " select p.name as profileName, pv.pageKey" + " from testavro.PAGEVIEW as pv" + " join testavro.PROFILE as p" + " on p.id = pv.profileId"; config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql); Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config)); List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList())); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig); QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig); translator.translate(queryInfo.get(0), streamAppDesc, 0); }
@Test (expected = SamzaException.class) public void testTranslateJoinWithIncorrectLeftJoin() { Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1); String sql = "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)" + " select p.name as profileName, pv.pageKey" + " from testavro.PAGEVIEW.`$table` as pv" + " left join testavro.PROFILE as p" + " on p.id = pv.profileId"; config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql); Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config)); List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList())); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig); QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig); translator.translate(queryInfo.get(0), streamAppDesc, 0); }
@Test (expected = SamzaException.class) public void testTranslateGroupByWithSumAggregator() { Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 10); String sql = "Insert into testavro.pageViewCountTopic(jobName, pageKey, `sum`)" + " select 'SampleJob' as jobName, pv.pageKey, sum(pv.profileId) as `sum`" + " from testavro.PAGEVIEW as pv" + " where pv.pageKey = 'job' or pv.pageKey = 'inbox'" + " group by (pv.pageKey)"; config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql); Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config)); List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList())); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig); QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig); translator.translate(queryInfo.get(0), streamAppDesc, 0); } }