@Override public Collection<RelRoot> convertDsl(String dsl) { // TODO: Introduce an API to parse a dsl string and return one or more sql statements List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig sqlConfig = new SamzaSqlApplicationConfig(config, queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toSet()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toSet())); QueryPlanner planner = new QueryPlanner(sqlConfig.getRelSchemaProviders(), sqlConfig.getSystemStreamConfigsBySource(), sqlConfig.getUdfMetadata()); List<RelRoot> relRoots = new LinkedList<>(); for (String sql: sqlStmts) { // when sql is a query, we only pass the select query to the planner SamzaSqlQueryParser.QueryInfo qinfo = SamzaSqlQueryParser.parseQuery(sql); if (qinfo.getSink().split("\\.")[0].equals(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG)) { sql = qinfo.getSelectQuery(); } relRoots.add(planner.plan(sql)); } return relRoots; }
@Override public Collection<RelRoot> convertDsl(String dsl) { // TODO: Introduce an API to parse a dsl string and return one or more sql statements List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig sqlConfig = new SamzaSqlApplicationConfig(config, queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList())); QueryPlanner planner = new QueryPlanner(sqlConfig.getRelSchemaProviders(), sqlConfig.getInputSystemStreamConfigBySource(), sqlConfig.getUdfMetadata()); List<RelRoot> relRoots = new LinkedList<>(); for (String sql: sqlStmts) { // we always pass only select query to the planner for samza sql. The reason is that samza sql supports // schema evolution where source and destination could up to an extent have independent schema evolution while // calcite expects strict comformance of the destination schema with that of the fields in the select query. SamzaSqlQueryParser.QueryInfo qinfo = SamzaSqlQueryParser.parseQuery(sql); relRoots.add(planner.plan(qinfo.getSelectQuery())); } return relRoots; }
/** * For unit testing only */ @VisibleForTesting void translate(SamzaSqlQueryParser.QueryInfo queryInfo, StreamApplicationDescriptor appDesc, int queryId) { QueryPlanner planner = new QueryPlanner(sqlConfig.getRelSchemaProviders(), sqlConfig.getInputSystemStreamConfigBySource(), sqlConfig.getUdfMetadata()); final RelRoot relRoot = planner.plan(queryInfo.getSelectQuery()); SamzaSqlExecutionContext executionContext = new SamzaSqlExecutionContext(sqlConfig); TranslatorContext translatorContext = new TranslatorContext(appDesc, relRoot, executionContext); translate(relRoot, sqlConfig.getOutputSystemStreams().get(queryId), translatorContext, queryId); Map<Integer, TranslatorContext> translatorContexts = new HashMap<>(); translatorContexts.put(queryId, translatorContext.clone()); appDesc.withApplicationTaskContextFactory(new ApplicationTaskContextFactory<SamzaSqlApplicationContext>() { @Override public SamzaSqlApplicationContext create(ExternalContext externalContext, JobContext jobContext, ContainerContext containerContext, TaskContext taskContext, ApplicationContainerContext applicationContainerContext) { return new SamzaSqlApplicationContext(translatorContexts); } }); }
/** * For unit testing only */ @VisibleForTesting public void translate(SamzaSqlQueryParser.QueryInfo queryInfo, StreamApplicationDescriptor appDesc) { QueryPlanner planner = new QueryPlanner(sqlConfig.getRelSchemaProviders(), sqlConfig.getSystemStreamConfigsBySource(), sqlConfig.getUdfMetadata()); final RelRoot relRoot = planner.plan(queryInfo.getSql()); int queryId = 1; SamzaSqlExecutionContext executionContext = new SamzaSqlExecutionContext(sqlConfig); Map<String, SamzaRelConverter> converters = sqlConfig.getSamzaRelConverters(); TranslatorContext translatorContext = new TranslatorContext(appDesc, relRoot, executionContext, converters); translate(relRoot, translatorContext, queryId); Map<Integer, TranslatorContext> translatorContexts = new HashMap<>(); translatorContexts.put(queryId, translatorContext.clone()); appDesc.withApplicationTaskContextFactory((jobContext, containerContext, taskContext, applicationContainerContext) -> new SamzaSqlApplicationContext(translatorContexts)); }