LOG.debug("Translating operator rel#" + joinRel.getId() + ":" + joinRel.getRelTypeName() + " with row type: [" + joinRel.getRowType() + "]");
LOG.debug("Translating operator rel#" + joinRel.getId() + ":" + joinRel.getRelTypeName() + " with row type: [" + joinRel.getRowType() + "]");
String indicator = "alwaysTrue" + e.rel.getId(); parentQueryFields.add(builder.alias(builder.literal(true), indicator)); builder.project(parentQueryFields); case TRUE: if (fields.isEmpty()) { builder.project(builder.alias(builder.literal(true), "i" + e.rel.getId())); if(!variablesSet.isEmpty() && (e.getKind() == SqlKind.EXISTS || e.getKind() == SqlKind.IN)) { fields.add(builder.alias(builder.literal(true), "i" + e.rel.getId())); builder.project(fields); builder.distinct(); break; operands.add(builder.isNotNull(builder.field("dt", "i" + e.rel.getId())), builder.literal(true)); if (!keyIsNulls.isEmpty()) {
String indicator = "alwaysTrue" + e.rel.getId(); parentQueryFields.add(builder.alias(builder.literal(true), indicator)); builder.project(parentQueryFields); case TRUE: if (fields.isEmpty()) { builder.project(builder.alias(builder.literal(true), "i" + e.rel.getId())); builder.aggregate(builder.groupKey(0)); } else { fields.add(builder.alias(builder.literal(true), "i" + e.rel.getId())); builder.project(fields); builder.distinct(); break; operands.add(builder.isNotNull(builder.field("dt", "i" + e.rel.getId())), builder.literal(true)); if (!keyIsNulls.isEmpty()) {
static String getStageName(RelNode n) { return n.getClass().getSimpleName().toUpperCase() + "_" + n.getId(); }
private void doChainOperators(PrintWriter pw, RelNode node, Set<TableScan> tables, String parentCtx) { pw.print( String.format(" ChannelContext CTX_%d = Channels.chain(%2$s, %3$s);\n", node.getId(), parentCtx, RelNodeCompiler.getStageName(node))); String currentCtx = String.format("CTX_%d", node.getId()); if (node instanceof TableScan) { tables.add((TableScan) node); } for (RelNode i : node.getInputs()) { doChainOperators(pw, i, tables, currentCtx); } }
public int compare(RelNode o1, RelNode o2) { // Compare on field count first. It is more stable than id (when rules // are added to the set of active rules). final int c = Utilities.compare(o1.getRowType().getFieldCount(), o2.getRowType().getFieldCount()); if (c != 0) { return -c; } return Utilities.compare(o1.getId(), o2.getId()); } }
public int compare(RelNode o1, RelNode o2) { // Compare on field count first. It is more stable than id (when rules // are added to the set of active rules). final int c = Utilities.compare(o1.getRowType().getFieldCount(), o2.getRowType().getFieldCount()); if (c != 0) { return -c; } return Utilities.compare(o1.getId(), o2.getId()); } }
void translate(final Project project, final String logicalOpId, final TranslatorContext context) { MessageStream<SamzaSqlRelMessage> messageStream = context.getMessageStream(project.getInput().getId()); List<Integer> flattenProjects = project.getProjects().stream().filter(this::isFlatten).map(this::getProjectIndex).collect(Collectors.toList()); if (flattenProjects.size() > 0) { if (flattenProjects.size() > 1) { String msg = "Multiple flatten operators in a single query is not supported"; LOG.error(msg); throw new SamzaException(msg); } messageStream = translateFlatten(flattenProjects.get(0), messageStream); } final int projectId = project.getId(); MessageStream<SamzaSqlRelMessage> outputStream = messageStream.map(new ProjectMapFunction(projectId, queryId, logicalOpId)); context.registerMessageStream(project.getId(), outputStream); context.registerRelNode(project.getId(), project); }
void translate(final Project project, final TranslatorContext context) { MessageStream<SamzaSqlRelMessage> messageStream = context.getMessageStream(project.getInput().getId()); List<Integer> flattenProjects = project.getProjects().stream().filter(this::isFlatten).map(this::getProjectIndex).collect(Collectors.toList()); if (flattenProjects.size() > 0) { if (flattenProjects.size() > 1) { String msg = "Multiple flatten operators in a single query is not supported"; LOG.error(msg); throw new SamzaException(msg); } messageStream = translateFlatten(flattenProjects.get(0), messageStream); } final int projectId = project.getId(); MessageStream<SamzaSqlRelMessage> outputStream = messageStream.map(new ProjectMapFunction(projectId, queryId)); context.registerMessageStream(project.getId(), outputStream); context.registerRelNode(project.getId(), project); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { double rowCount = mq.getRowCount(this); // Joins can be flipped, and for many algorithms, both versions are viable // and have the same cost. To make the results stable between versions of // the planner, make one of the versions slightly more expensive. switch (joinType) { case RIGHT: rowCount = addEpsilon(rowCount); break; default: if (left.getId() > right.getId()) { rowCount = addEpsilon(rowCount); } } final double rightRowCount = right.estimateRowCount(mq); final double leftRowCount = left.estimateRowCount(mq); if (Double.isInfinite(leftRowCount)) { rowCount = leftRowCount; } if (Double.isInfinite(rightRowCount)) { rowCount = rightRowCount; } return planner.getCostFactory().makeCost(rowCount, 0, 0); }
@Override public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) { double rowCount = mq.getRowCount(this); // Joins can be flipped, and for many algorithms, both versions are viable // and have the same cost. To make the results stable between versions of // the planner, make one of the versions slightly more expensive. switch (joinType) { case RIGHT: rowCount = addEpsilon(rowCount); break; default: if (left.getId() > right.getId()) { rowCount = addEpsilon(rowCount); } } final double rightRowCount = right.estimateRowCount(mq); final double leftRowCount = left.estimateRowCount(mq); if (Double.isInfinite(leftRowCount)) { rowCount = leftRowCount; } if (Double.isInfinite(rightRowCount)) { rowCount = rightRowCount; } return planner.getCostFactory().makeCost(rowCount, 0, 0); }
void translate(final LogicalFilter filter, final String logicalOpId, final TranslatorContext context) { MessageStream<SamzaSqlRelMessage> inputStream = context.getMessageStream(filter.getInput().getId()); final int filterId = filter.getId(); MessageStream<SamzaSqlRelMessage> outputStream = inputStream.filter(new FilterTranslatorFunction(filterId, queryId, logicalOpId)); context.registerMessageStream(filterId, outputStream); context.registerRelNode(filterId, filter); } }
void translate(final LogicalFilter filter, final TranslatorContext context) { MessageStream<SamzaSqlRelMessage> inputStream = context.getMessageStream(filter.getInput().getId()); final int filterId = filter.getId(); MessageStream<SamzaSqlRelMessage> outputStream = inputStream.filter(new FilterTranslatorFunction(filterId, queryId)); context.registerMessageStream(filterId, outputStream); context.registerRelNode(filterId, filter); } }
private Table loadLocalTable(boolean isTablePosOnRight, List<Integer> tableKeyIds, LogicalJoin join, TranslatorContext context) { RelNode relNode = isTablePosOnRight ? join.getRight() : join.getLeft(); MessageStream<SamzaSqlRelMessage> relOutputStream = context.getMessageStream(relNode.getId()); SqlIOConfig sourceConfig = resolveSourceConfig(relNode); if (!sourceConfig.getTableDescriptor().isPresent()) { String errMsg = "Failed to resolve table source in join operation: node=" + relNode; log.error(errMsg); throw new SamzaException(errMsg); } // Create a table backed by RocksDb store with the fields in the join condition as composite key and relational // message as the value. Send the messages from the input stream denoted as 'table' to the created table store. Table<KV<SamzaSqlCompositeKey, SamzaSqlRelMessage>> table = context.getStreamAppDescriptor().getTable(sourceConfig.getTableDescriptor().get()); relOutputStream .map(m -> new KV(createSamzaSqlCompositeKey(m, tableKeyIds), m)) .sendTo(table); return table; } }
private void sendToOutputStream(String queryLogicalId, String logicalOpId, String sinkStream, StreamApplicationDescriptor appDesc, TranslatorContext translatorContext, RelNode node, int queryId) { SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(sinkStream); MessageStream<SamzaSqlRelMessage> stream = translatorContext.getMessageStream(node.getId()); MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(queryLogicalId, logicalOpId, sinkStream, queryId)); Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor(); if (!tableDescriptor.isPresent()) { KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()); String systemName = sinkConfig.getSystemName(); DelegatingSystemDescriptor sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new); GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamId(), noOpKVSerde); OutputStream stm = outputMsgStreams.computeIfAbsent(sinkConfig.getSource(), v -> appDesc.getOutputStream(osd)); outputStream.sendTo(stm); } else { Table outputTable = appDesc.getTable(tableDescriptor.get()); if (outputTable == null) { String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource(); throw new SamzaException(msg); } outputStream.sendTo(outputTable); } } }
private void sendToOutputStream(StreamApplicationDescriptor appDesc, TranslatorContext context, RelNode node, int queryId) { SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG); MessageStream<SamzaSqlRelMessage> stream = context.getMessageStream(node.getId()); MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG, queryId)); Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor(); if (!tableDescriptor.isPresent()) { KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()); String systemName = sinkConfig.getSystemName(); DelegatingSystemDescriptor sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new); GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamName(), noOpKVSerde); if (OutputMapFunction.logOutputStream == null) { OutputMapFunction.logOutputStream = appDesc.getOutputStream(osd); } outputStream.sendTo(OutputMapFunction.logOutputStream); } else { Table outputTable = appDesc.getTable(tableDescriptor.get()); if (outputTable == null) { String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource(); throw new SamzaException(msg); } outputStream.sendTo(outputTable); } } }
void translate(final LogicalAggregate aggregate, final TranslatorContext context) { validateAggregateFunctions(aggregate); MessageStream<SamzaSqlRelMessage> inputStream = context.getMessageStream(aggregate.getInput().getId()); // At this point, the assumption is that only count function is supported. SupplierFunction<Long> initialValue = () -> (long) 0; FoldLeftFunction<SamzaSqlRelMessage, Long> foldCountFn = (m, c) -> c + 1; final ArrayList<String> aggFieldNames = getAggFieldNames(aggregate); MessageStream<SamzaSqlRelMessage> outputStream = inputStream .window(Windows.keyedTumblingWindow(m -> m, Duration.ofMillis(context.getExecutionContext().getSamzaSqlApplicationConfig().getWindowDurationMs()), initialValue, foldCountFn, new SamzaSqlRelMessageSerdeFactory.SamzaSqlRelMessageSerde(), new LongSerde()) .setAccumulationMode(AccumulationMode.DISCARDING), "tumblingWindow_" + windowId) .map(windowPane -> { List<String> fieldNames = windowPane.getKey().getKey().getSamzaSqlRelRecord().getFieldNames(); List<Object> fieldValues = windowPane.getKey().getKey().getSamzaSqlRelRecord().getFieldValues(); fieldNames.add(aggFieldNames.get(0)); fieldValues.add(windowPane.getMessage()); return new SamzaSqlRelMessage(fieldNames, fieldValues); }); context.registerMessageStream(aggregate.getId(), outputStream); }
void translate(final LogicalAggregate aggregate, final TranslatorContext context) { validateAggregateFunctions(aggregate); MessageStream<SamzaSqlRelMessage> inputStream = context.getMessageStream(aggregate.getInput().getId()); // At this point, the assumption is that only count function is supported. SupplierFunction<Long> initialValue = () -> (long) 0; FoldLeftFunction<SamzaSqlRelMessage, Long> foldCountFn = (m, c) -> c + 1; final ArrayList<String> aggFieldNames = getAggFieldNames(aggregate); MessageStream<SamzaSqlRelMessage> outputStream = inputStream .map(new TranslatorInputMetricsMapFunction(logicalOpId)) .window(Windows.keyedTumblingWindow(m -> m, Duration.ofMillis(context.getExecutionContext().getSamzaSqlApplicationConfig().getWindowDurationMs()), initialValue, foldCountFn, new SamzaSqlRelMessageSerdeFactory.SamzaSqlRelMessageSerde(), new LongSerde()) .setAccumulationMode( AccumulationMode.DISCARDING), changeLogStorePrefix + "_tumblingWindow_" + logicalOpId) .map(windowPane -> { List<String> fieldNames = windowPane.getKey().getKey().getSamzaSqlRelRecord().getFieldNames(); List<Object> fieldValues = windowPane.getKey().getKey().getSamzaSqlRelRecord().getFieldValues(); fieldNames.add(aggFieldNames.get(0)); fieldValues.add(windowPane.getMessage()); return new SamzaSqlRelMessage(fieldNames, fieldValues, new SamzaSqlRelMsgMetadata("", "", "")); }); context.registerMessageStream(aggregate.getId(), outputStream); outputStream.map(new TranslatorOutputMetricsMapFunction(logicalOpId)); }
void translate(final LogicalJoin join, final TranslatorContext translatorContext) { JoinInputNode.InputType inputTypeOnLeft = getInputType(join.getLeft(), translatorContext); JoinInputNode.InputType inputTypeOnRight = getInputType(join.getRight(), translatorContext); // Do the validation of join query validateJoinQuery(join, inputTypeOnLeft, inputTypeOnRight); // At this point, one of the sides is a table. Let's figure out if it is on left or right side. boolean isTablePosOnRight = (inputTypeOnRight != JoinInputNode.InputType.STREAM); // stream and table keyIds are used to extract the join condition field (key) names and values out of the stream // and table records. List<Integer> streamKeyIds = new LinkedList<>(); List<Integer> tableKeyIds = new LinkedList<>(); // Fetch the stream and table indices corresponding to the fields given in the join condition. populateStreamAndTableKeyIds(((RexCall) join.getCondition()).getOperands(), join, isTablePosOnRight, streamKeyIds, tableKeyIds); // Get the two input nodes (stream and table nodes) for the join. JoinInputNode streamNode = new JoinInputNode(isTablePosOnRight ? join.getLeft() : join.getRight(), streamKeyIds, isTablePosOnRight ? inputTypeOnLeft : inputTypeOnRight, !isTablePosOnRight); JoinInputNode tableNode = new JoinInputNode(isTablePosOnRight ? join.getRight() : join.getLeft(), tableKeyIds, isTablePosOnRight ? inputTypeOnRight : inputTypeOnLeft, isTablePosOnRight); MessageStream<SamzaSqlRelMessage> inputStream = translatorContext.getMessageStream(streamNode.getRelNode().getId()); Table table = getTable(tableNode, translatorContext); MessageStream<SamzaSqlRelMessage> outputStream = joinStreamWithTable(inputStream, table, streamNode, tableNode, join, translatorContext); translatorContext.registerMessageStream(join.getId(), outputStream); outputStream.map(outputMetricsMF); }