How to use

SamzaSqlApplicationRunner

in

org.apache.samza.sql.runner

private SamzaSqlApplicationRunner(SamzaApplication app, Boolean localRunner, Config config) {
 this.runner = ApplicationRunners.getApplicationRunner(app, computeSamzaConfigs(localRunner, config));
}

public void runAndWaitForFinish(ExternalContext externalContext) {
 Validate.isTrue(runner instanceof LocalApplicationRunner, "This method can be called only in standalone mode.");
 run(externalContext);
 waitForFinish();
}

@Override
public NonQueryResult executeNonQuery(ExecutionContext context, List<String> statement) {
 lastErrorMsg = "";
 int execId = execIdSeq.incrementAndGet();
 Map<String, String> staticConfigs = fetchSamzaSqlConfig(execId, context);
 staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(formatSqlStmts(statement)));
 SamzaSqlApplicationRunner runner;
 try {
  runner = new SamzaSqlApplicationRunner(true, new MapConfig(staticConfigs));
  runner.run(null);
 } catch (SamzaException ex) {
  String msg = "Execution of the query failed with exception ";
  lastErrorMsg = msg + ex.toString();
  LOG.error(msg, ex);
  return new NonQueryResult(execId, false);
 }
 executions.put(execId, runner);
 LOG.debug("Executing sql. Id ", execId);
 return new NonQueryResult(execId, true);
}

@Override
public boolean removeExecution(ExecutionContext context, int exeId) {
 lastErrorMsg = "";
 SamzaSqlApplicationRunner runner = executions.get(exeId);
 if (runner != null) {
  if (runner.status().getStatusCode().equals(ApplicationStatus.StatusCode.Running)) {
   lastErrorMsg = "Trying to remove a ongoing execution " + exeId;
   LOG.error(lastErrorMsg);
   return false;
  }
  executions.remove(exeId);
  LOG.debug("Stopping execution ", exeId);
  return true;
 } else {
  lastErrorMsg = "Trying to remove a non-existing SQL execution " + exeId;
  LOG.warn(lastErrorMsg);
  return false;
 }
}

public void runAndWaitForFinish() {
 runAndWaitForFinish(null);
}

@Override
public boolean stopExecution(ExecutionContext context, int exeId) {
 lastErrorMsg = "";
 SamzaSqlApplicationRunner runner = executions.get(exeId);
 if (runner != null) {
  LOG.debug("Stopping execution ", exeId);
  try {
   runner.kill();
   } catch (SamzaException ex) {
   String msg = "Stopping execution failed with exception ";
   lastErrorMsg = msg + ex.toString();
   LOG.warn(msg, ex);
   return false;
  }
  try {
   Thread.sleep(500); // wait for a second
  } catch (InterruptedException e) {
   e.printStackTrace();
  }
  return true;
 } else {
  lastErrorMsg = "Trying to stop a non-existing SQL execution " + exeId;
  LOG.warn(lastErrorMsg);
  return false;
 }
}

private ExecutionStatus queryExecutionStatus(SamzaSqlApplicationRunner runner) {
 lastErrorMsg = "";
 switch (runner.status().getStatusCode()) {
  case New:
   return ExecutionStatus.New;
  case Running:
   return ExecutionStatus.Running;
  case SuccessfulFinish:
   return ExecutionStatus.SuccessfulFinish;
  case UnsuccessfulFinish:
   return ExecutionStatus.UnsuccessfulFinish;
  default:
   lastErrorMsg = String.format("Unsupported execution status %s",
       runner.status().getStatusCode().toString());
   return null;
 }
}

private SamzaSqlApplicationRunner(SamzaApplication app, Boolean localRunner, Config config) {
 this.runner = ApplicationRunners.getApplicationRunner(app, computeSamzaConfigs(localRunner, config));
}

public void runAndWaitForFinish() {
 Validate.isTrue(runner instanceof LocalApplicationRunner, "This method can be called only in standalone mode.");
 run();
 waitForFinish();
}

@Override
public QueryResult executeQuery(ExecutionContext context, String statement) {
 lastErrorMsg = "";
 outputData.clear();
 int execId = execIdSeq.incrementAndGet();
 Map<String, String> staticConfigs = fetchSamzaSqlConfig(execId, context);
 List<String> sqlStmts = formatSqlStmts(Collections.singletonList(statement));
 staticConfigs.put(SamzaSqlApplicationConfig.CFG_SQL_STMTS_JSON, JsonUtil.toJson(sqlStmts));
 SamzaSqlApplicationRunner runner;
 try {
  runner = new SamzaSqlApplicationRunner(true, new MapConfig(staticConfigs));
  runner.run(null);
 } catch (SamzaException ex) {
  String msg = "Execution failed with exception ";
  lastErrorMsg = msg + ex.toString();
  LOG.error(msg, ex);
  return new QueryResult(execId, null, false);
 }
 executions.put(execId, runner);
 LOG.debug("Executing sql. Id ", execId);
 return new QueryResult(execId, generateResultSchema(new MapConfig(staticConfigs)), true);
}

 @Test
 public void testComputeSamzaConfigs() {
  Map<String, String> configs = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10);
  String sql1 = "Insert into testavro.outputTopic(id,long_value) select id, MyTest(id) as long_value from testavro.SIMPLE1";
  configs.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql1);
  configs.put(SamzaSqlApplicationRunner.RUNNER_CONFIG, SamzaSqlApplicationRunner.class.getName());
  MapConfig samzaConfig = new MapConfig(configs);
  Config newConfigs = SamzaSqlApplicationRunner.computeSamzaConfigs(true, samzaConfig);
  Assert.assertEquals(newConfigs.get(SamzaSqlApplicationRunner.RUNNER_CONFIG), LocalApplicationRunner.class.getName());
  // Check whether five new configs added.
  Assert.assertEquals(newConfigs.size(), configs.size() + 5);

  newConfigs = SamzaSqlApplicationRunner.computeSamzaConfigs(false, samzaConfig);
  Assert.assertEquals(newConfigs.get(SamzaSqlApplicationRunner.RUNNER_CONFIG), RemoteApplicationRunner.class.getName());

  // Check whether five new configs added.
  Assert.assertEquals(newConfigs.size(), configs.size() + 5);
 }
}

@Test (expected = SamzaException.class)
public void testTranslateStreamTableCrossJoin() {
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1);
 String sql =
   "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)"
     + " select p.name as profileName, pv.pageKey"
     + " from testavro.PAGEVIEW as pv, testavro.PROFILE.`$table` as p";
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
 Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
 List<String> sqlStmts = fetchSqlFromConfig(config);
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig);
 QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
 translator.translate(queryInfo.get(0), streamAppDesc, 0);
}

@Test (expected = SamzaException.class)
public void testTranslateStreamTableInnerJoinWithMissingStream() {
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 10);
 String configIOResolverDomain =
   String.format(SamzaSqlApplicationConfig.CFG_FMT_SOURCE_RESOLVER_DOMAIN, "config");
 config.put(configIOResolverDomain + SamzaSqlApplicationConfig.CFG_FACTORY,
   ConfigBasedIOResolverFactory.class.getName());
 String sql =
   "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)"
     + " select p.name as profileName, pv.pageKey"
     + " from testavro.PAGEVIEW as pv"
     + " join testavro.`$table` as p"
     + " on p.id = pv.profileId";
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
 Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
 List<String> sqlStmts = fetchSqlFromConfig(config);
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig);
 QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
 translator.translate(queryInfo.get(0), streamAppDesc, 0);
}

@Test (expected = SamzaException.class)
public void testTranslateStreamTableJoinWithSubQuery() {
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1);
 String sql =
   "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)"
     + " select p.name as profileName, pv.pageKey"
     + " from testavro.PAGEVIEW as pv"
     + " where exists "
     + " (select p.id from testavro.PROFILE.`$table` as p"
     + " where p.id = pv.profileId)";
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
 Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
 List<String> sqlStmts = fetchSqlFromConfig(config);
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig);
 QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
 translator.translate(queryInfo.get(0), streamAppDesc, 0);
}

@Test (expected = SamzaException.class)
public void testTranslateStreamTableJoinWithoutJoinOperator() {
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1);
 String sql =
   "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)"
     + " select p.name as profileName, pv.pageKey"
     + " from testavro.PAGEVIEW as pv, testavro.PROFILE.`$table` as p"
     + " where p.id = pv.profileId";
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
 Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
 List<String> sqlStmts = fetchSqlFromConfig(config);
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig);
 QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
 translator.translate(queryInfo.get(0), streamAppDesc, 0);
}

@Test (expected = SamzaException.class)
public void testTranslateStreamTableJoinWithFullJoinOperator() {
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1);
 String sql =
   "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)"
     + " select p.name as profileName, pv.pageKey"
     + " from testavro.PAGEVIEW as pv"
     + " full join testavro.PROFILE.`$table` as p"
     + " on p.id = pv.profileId";
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
 Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
 List<String> sqlStmts = fetchSqlFromConfig(config);
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig);
 QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
 translator.translate(queryInfo.get(0), streamAppDesc, 0);
}

@Test (expected = SamzaException.class)
public void testTranslateTableTableJoin() {
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1);
 String sql =
   "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)"
     + " select p.name as profileName, pv.pageKey"
     + " from testavro.PAGEVIEW.`$table` as pv"
     + " join testavro.PROFILE.`$table` as p"
     + " on p.id = pv.profileId";
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
 Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
 List<String> sqlStmts = fetchSqlFromConfig(config);
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig);
 QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
 translator.translate(queryInfo.get(0), streamAppDesc, 0);
}

@Test (expected = SamzaException.class)
public void testTranslateStreamStreamJoin() {
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1);
 String sql =
   "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)"
     + " select p.name as profileName, pv.pageKey"
     + " from testavro.PAGEVIEW as pv"
     + " join testavro.PROFILE as p"
     + " on p.id = pv.profileId";
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
 Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
 List<String> sqlStmts = fetchSqlFromConfig(config);
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig);
 QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
 translator.translate(queryInfo.get(0), streamAppDesc, 0);
}

@Test (expected = SamzaException.class)
public void testTranslateJoinWithIncorrectLeftJoin() {
 Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 1);
 String sql =
   "Insert into testavro.enrichedPageViewTopic(profileName, pageKey)"
     + " select p.name as profileName, pv.pageKey"
     + " from testavro.PAGEVIEW.`$table` as pv"
     + " left join testavro.PROFILE as p"
     + " on p.id = pv.profileId";
 config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
 Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));
 List<String> sqlStmts = fetchSqlFromConfig(config);
 List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
 SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
     .collect(Collectors.toList()),
   queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));
 StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig);
 QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
 translator.translate(queryInfo.get(0), streamAppDesc, 0);
}

 @Test (expected = SamzaException.class)
 public void testTranslateGroupByWithSumAggregator() {
  Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(configs, 10);
  String sql =
    "Insert into testavro.pageViewCountTopic(jobName, pageKey, `sum`)"
      + " select 'SampleJob' as jobName, pv.pageKey, sum(pv.profileId) as `sum`"
      + " from testavro.PAGEVIEW as pv" + " where pv.pageKey = 'job' or pv.pageKey = 'inbox'"
      + " group by (pv.pageKey)";
  config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, sql);
  Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config));

  List<String> sqlStmts = fetchSqlFromConfig(config);
  List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts);
  SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config),
    queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream)
      .collect(Collectors.toList()),
    queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList()));

  StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig);
  QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig);
  translator.translate(queryInfo.get(0), streamAppDesc, 0);
 }
}