PlanNormalizer(SqlHandlerConfig sqlHandlerConfig) { this.optionManager = sqlHandlerConfig.getContext().getOptions(); }
public ReflectionPlanNormalizer( SqlHandlerConfig sqlHandlerConfig, ReflectionGoal goal, ReflectionEntry entry, Materialization materialization, NamespaceService namespace, SabotConfig config, ReflectionSettings reflectionSettings, MaterializationStore materializationStore) { this.sqlHandlerConfig = sqlHandlerConfig; this.goal = goal; this.entry = entry; this.materialization = materialization; this.namespace = namespace; this.config = config; this.reflectionSettings = reflectionSettings; this.materializationStore = materializationStore; this.optionManager = sqlHandlerConfig.getContext().getOptions(); }
public static void log(final SqlHandlerConfig config, final String name, final PhysicalPlan plan, final Logger logger) throws JsonProcessingException { if (logger.isDebugEnabled()) { String planText = plan.unparse(config.getContext().getLpPersistence().getMapper().writer()); logger.debug(name + " : \n" + planText); } }
public static PhysicalOperator convertToPop(SqlHandlerConfig config, Prel prel) throws IOException { PhysicalPlanCreator creator = new PhysicalPlanCreator(config.getContext(), PrelSequencer.getIdMap(prel)); PhysicalOperator op = prel.getPhysicalOperator(creator); return op; }
public ExplainHandler(SqlHandlerConfig config) { super(); this.config = new SqlHandlerConfig(config.getContext(), config.getConverter(), config.getObserver(), config.getMaterializations().orNull()); }
private static RelNode preprocessNode(SqlHandlerConfig config, RelNode rel) throws SqlUnsupportedException { /* * Traverse the tree to do the following pre-processing tasks: 1. replace the convert_from, convert_to function to * actual implementations Eg: convert_from(EXPR, 'JSON') be converted to convert_fromjson(EXPR); TODO: Ideally all * function rewrites would move here instead of RexToExpr. * * 2. see where the tree contains unsupported functions; throw SqlUnsupportedException if there is any. */ PreProcessRel visitor = PreProcessRel.createVisitor( config.getContext().getOperatorTable(), rel.getCluster().getRexBuilder()); try { rel = rel.accept(visitor); } catch (UnsupportedOperationException ex) { visitor.convertException(); throw ex; } return rel; }
@Override public Prel visitPrel(Prel prel, Void value) throws IOException { if (prel instanceof LimitPrel) { LimitPrel limit = (LimitPrel) prel; if(isLimit0(limit.getFetch())){ PhysicalOperator op = PrelTransformer.convertToPop(config, prel); BatchSchema schema = op.getSchema(config.getContext().getFunctionRegistry()); // make sure to remove any selection vector modes since we're now the leaf node. schema = schema.clone(SelectionVectorMode.NONE); return new EmptyPrel(prel.getCluster(), prel.getTraitSet(), prel.getRowType(), schema); } } List<RelNode> children = new ArrayList<>(); for(Prel child : prel){ children.add(child.accept(this, null)); } return (Prel) prel.copy(prel.getTraitSet(), children); }
public static PhysicalPlan convertToPlan(SqlHandlerConfig config, PhysicalOperator op) { PlanPropertiesBuilder propsBuilder = PlanProperties.builder(); propsBuilder.type(PlanType.PHYSICAL); propsBuilder.version(1); propsBuilder.options(new JSONOptions(config.getContext().getOptions().getOptionList())); propsBuilder.resultMode(ResultMode.EXEC); propsBuilder.generator("default", "handler"); List<PhysicalOperator> ops = Lists.newArrayList(); PopCollector c = new PopCollector(); op.accept(c, ops); return new PhysicalPlan(propsBuilder.build(), ops); }
/** * Return Dremio Logical RelNode tree for a SELECT statement, when it is executed / explained directly. * * @param relNode : root RelNode corresponds to Calcite Logical RelNode. * @param validatedRowType : the rowType for the final field names. A rename project may be placed on top of the root. * @return * @throws RelConversionException * @throws SqlUnsupportedException */ public static Rel convertToDrel(SqlHandlerConfig config, RelNode relNode, RelDataType validatedRowType) throws RelConversionException, SqlUnsupportedException { Rel convertedRelNode = convertToDrel(config, relNode); // Put a non-trivial topProject to ensure the final output field name is preserved, when necessary. convertedRelNode = addRenamedProject(config, convertedRelNode, validatedRowType); convertedRelNode = SqlHandlerUtil.storeQueryResultsIfNeeded(config.getConverter().getParserConfig(), config.getContext(), convertedRelNode); return new ScreenRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode); }
private static Pair<SqlNode, RelDataType> validateNode(SqlHandlerConfig config, final SqlNode sqlNode) throws ValidationException, RelConversionException, ForemanSetupException { final Stopwatch stopwatch = Stopwatch.createStarted(); final SqlNode sqlNodeValidated; try { sqlNodeValidated = config.getConverter().validate(sqlNode); } catch (final Throwable ex) { throw new ValidationException("unable to validate sql node", ex); } final Pair<SqlNode, RelDataType> typedSqlNode = new Pair<>(sqlNodeValidated, config.getConverter().getOutputType(sqlNodeValidated)); // Check if the unsupported functionality is used UnsupportedOperatorsVisitor visitor = UnsupportedOperatorsVisitor.createVisitor(config.getContext()); try { sqlNodeValidated.accept(visitor); } catch (UnsupportedOperationException ex) { // If the exception due to the unsupported functionalities visitor.convertException(); // If it is not, let this exception move forward to higher logic throw ex; } config.getObserver().planValidated(typedSqlNode.getValue(), typedSqlNode.getKey(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); return typedSqlNode; }
final RelNode intermediateNode = transform(config, PlannerType.VOLCANO, PlannerPhase.LOGICAL, preLog, logicalTraits, true); final Catalog catalog = config.getContext().getCatalog(); if (catalog instanceof CachingCatalog) { config.getObserver().tablesCollected(catalog.getAllRequestedTables());
final SqlCompactMaterialization compact = SqlNodeUtil.unwrap(sqlNode, SqlCompactMaterialization.class); if(!SystemUser.SYSTEM_USERNAME.equals(config.getContext().getQueryUserName())) { throw SqlExceptionHelper.parseError("$COMPACT REFRESH not supported.", sql, compact.getParserPosition()) .build(logger); ReflectionService service = config.getContext().getAccelerationManager().unwrap(ReflectionService.class); final long ringCount = config.getContext().getOptions().getOption(PlannerSettings.RING_COUNT); final Rel writerDrel = new WriterRel(drel.getCluster(), drel.getCluster().traitSet().plus(Rel.LOGICAL), drel, config.getContext().getCatalog().createNewTable( new NamespaceKey(ReflectionUtils.getMaterializationPath(newMaterialization)), getWriterOptions((int) ringCount, goal, fields), ImmutableMap.of()), ); final RelNode doubleWriter = SqlHandlerUtil.storeQueryResultsIfNeeded(config.getConverter().getParserConfig(), config.getContext(), writerDrel);
final SqlRefreshReflection materialize = SqlNodeUtil.unwrap(sqlNode, SqlRefreshReflection.class); if(!SystemUser.SYSTEM_USERNAME.equals(config.getContext().getQueryUserName())) { throw SqlExceptionHelper.parseError("$MATERIALIZE not supported.", sql, materialize.getParserPosition()).build(logger); ReflectionService service = config.getContext().getAccelerationManager().unwrap(ReflectionService.class); namespace, new ExtendedToRelContext(config.getConverter()), config.getContext().getConfig(), reflectionSettings, materializationStore); final UserBitShared.QueryId queryId = config.getContext().getQueryId(); final AttemptId attemptId = AttemptId.of(queryId); drel, config.getContext().getCatalog().createNewTable( new NamespaceKey(tablePath), getWriterOptions(0, goal, fields), ImmutableMap.of() ), initial.getRowType()); final RelNode doubleWriter = SqlHandlerUtil.storeQueryResultsIfNeeded(config.getConverter().getParserConfig(), config.getContext(), writerDrel);
final RelNode queryRelNode = convertedRelNode.getConvertedNode(); final RelNode newViewRelNode = SqlHandlerUtil.resolveNewTableRel(true, createView.getFieldNames(), validatedRowType, queryRelNode); Catalog catalog = config.getContext().getCatalog(); NamespaceKey viewPath = catalog.resolveSingle(createView.getPath()); NamespaceKey defaultSchema = catalog.getDefaultSchema(); final DremioTable existingTable = config.getContext().getCatalog().getTableNoResolve(viewPath); List<String> viewContext = defaultSchema == null ? null : defaultSchema.getPathComponents(); config.getContext().getCatalog().updateView(viewPath, view); replaced = true; } else { config.getContext().getCatalog().createView(viewPath, view);
private static Rel convertToDrel( SqlHandlerConfig config, RelNode relNode, Catalog catalog, NamespaceKey key, WriterOptions options, RelDataType queryRowType, final Map<String, Object> storageOptions) throws RelConversionException, SqlUnsupportedException { Rel convertedRelNode = PrelTransformer.convertToDrel(config, relNode); // Put a non-trivial topProject to ensure the final output field name is preserved, when necessary. // Only insert project when the field count from the child is same as that of the queryRowType. convertedRelNode = new WriterRel(convertedRelNode.getCluster(), convertedRelNode.getCluster().traitSet().plus(Rel.LOGICAL), convertedRelNode, catalog.createNewTable(key, options, storageOptions), queryRowType); convertedRelNode = SqlHandlerUtil.storeQueryResultsIfNeeded(config.getConverter().getParserConfig(), config.getContext(), convertedRelNode); return new ScreenRel(convertedRelNode.getCluster(), convertedRelNode.getTraitSet(), convertedRelNode); }
Iterable<DremioTable> requestedTables = sqlHandlerConfig.getContext().getCatalog().getAllRequestedTables();
final HepPlanner hepPlanner = new DremioHepPlanner(hepPgmBldr.build(), config.getContext().getPlannerSettings(), converter.getCostFactory(), phase); hepPlanner.setExecutor(new ConstExecutor(converter.getFunctionImplementationRegistry(), converter.getFunctionContext(), converter.getSettings()));
final Catalog catalog = config.getContext().getCatalog(); final NamespaceKey path = catalog.resolveSingle(sqlCreateTable.getPath()); DremioTable table = catalog.getTableNoResolve(path); final RelNode newTblRelNode = SqlHandlerUtil.resolveNewTableRel(false, sqlCreateTable.getFieldNames(), validatedRowType, queryRelNode); final long ringCount = config.getContext().getOptions().getOption(PlannerSettings.RING_COUNT);
final String postPhysicaltextPlan = plan.unparse(config.getContext().getLpPersistence().getMapper().writer());
final boolean leafLimitEnabled = config.getContext().getPlannerSettings().isLeafLimitsEnabled();