/** * Creates a new contract with the given name. The parameters are empty by default and * the compiler hints are not set. * * @param name The name that is used to describe the contract. */ protected Operator(OperatorInformation<OUT> operatorInfo, String name) { this.name = (name == null) ? "(null)" : name; this.parameters = new Configuration(); this.compilerHints = new CompilerHints(); this.operatorInfo = operatorInfo; }
if (hints.getOutputSize() >= 0) { this.estimatedOutputSize = hints.getOutputSize(); if (hints.getOutputCardinality() >= 0) { this.estimatedNumRecords = hints.getOutputCardinality(); if (hints.getFilterFactor() >= 0.0f) { if (this.estimatedNumRecords >= 0) { this.estimatedNumRecords = (long) (this.estimatedNumRecords * hints.getFilterFactor()); this.estimatedOutputSize = (long) (this.estimatedOutputSize * hints.getFilterFactor()); OptimizerNode pred = ((SingleInputNode) this).getPredecessorNode(); if (pred != null && pred.getEstimatedNumRecords() >= 0) { this.estimatedNumRecords = (long) (pred.getEstimatedNumRecords() * hints.getFilterFactor()); if (hints.getAvgOutputRecordSize() >= 1) { this.estimatedNumRecords = (long) (this.estimatedOutputSize / hints.getAvgOutputRecordSize()); this.estimatedOutputSize = (long) (this.estimatedNumRecords * hints.getAvgOutputRecordSize());
protected void readUniqueFieldsAnnotation() { if (this.pactContract.getCompilerHints() != null) { Set<FieldSet> uniqueFieldSets = pactContract.getCompilerHints().getUniqueFields(); if (uniqueFieldSets != null) { if (this.uniqueFields == null) { this.uniqueFields = new HashSet<FieldSet>(); } this.uniqueFields.addAll(uniqueFieldSets); } } }
@Override public float getAvgOutputRecordSize() { float superWidth = super.getAvgOutputRecordSize(); if (superWidth > 0.0f || this.width <= 0.0f) { return superWidth; } else { return this.width; } }
setSourceStatistics(ordersSource, orderSize, 100f); setSourceStatistics(lineItemSource, lineitemSize, 140f); mapper.getCompilerHints().setAvgOutputRecordSize(16f); mapper.getCompilerHints().setFilterFactor(orderSelectivity); joiner.getCompilerHints().setFilterFactor(joinSelectivity);
/** * Statistics that push towards a repartition merge join. If the join blows the data volume up significantly, * re-exploiting the sorted order is cheaper. */ @Test public void testQueryWithStatsForRepartitionMerge() { TPCHQuery3 query = new TPCHQuery3(); Plan p = query.getPlan(DEFAULT_PARALLELISM_STRING, IN_FILE, IN_FILE, OUT_FILE); // set compiler hints OperatorResolver cr = getContractResolver(p); JoinOperator match = cr.getNode("JoinLiO"); match.getCompilerHints().setFilterFactor(100f); testQueryGeneric(100l*1024*1024*1024*1024, 100l*1024*1024*1024*1024, 0.05f, 100f, false, true, false, false, true); }
.name("Filter Docs") .build(); filterDocs.getCompilerHints().setFilterFactor(0.15f); .name("Filter Ranks") .build(); filterRanks.getCompilerHints().setFilterFactor(0.25f); .name("Filter Visits") .build(); filterVisits.getCompilerHints().setFilterFactor(0.2f);
if (hints.getOutputSize() >= 0) { this.estimatedOutputSize = hints.getOutputSize(); if (hints.getOutputCardinality() >= 0) { this.estimatedNumRecords = hints.getOutputCardinality(); if (hints.getFilterFactor() >= 0.0f) { if (this.estimatedNumRecords >= 0) { this.estimatedNumRecords = (long) (this.estimatedNumRecords * hints.getFilterFactor()); this.estimatedOutputSize = (long) (this.estimatedOutputSize * hints.getFilterFactor()); OptimizerNode pred = ((SingleInputNode) this).getPredecessorNode(); if (pred != null && pred.getEstimatedNumRecords() >= 0) { this.estimatedNumRecords = (long) (pred.getEstimatedNumRecords() * hints.getFilterFactor()); if (hints.getAvgOutputRecordSize() >= 1) { this.estimatedNumRecords = (long) (this.estimatedOutputSize / hints.getAvgOutputRecordSize()); this.estimatedOutputSize = (long) (this.estimatedNumRecords * hints.getAvgOutputRecordSize());
/** * Creates a new contract with the given name. The parameters are empty by default and * the compiler hints are not set. * * @param name The name that is used to describe the contract. */ protected Operator(OperatorInformation<OUT> operatorInfo, String name) { this.name = (name == null) ? "(null)" : name; this.parameters = new Configuration(); this.compilerHints = new CompilerHints(); this.operatorInfo = operatorInfo; }
filterO.setParameter(PRIO_FILTER, "5"); filterO.getCompilerHints().setFilterFactor(0.05f);
protected void readUniqueFieldsAnnotation() { if (this.pactContract.getCompilerHints() != null) { Set<FieldSet> uniqueFieldSets = pactContract.getCompilerHints().getUniqueFields(); if (uniqueFieldSets != null) { if (this.uniqueFields == null) { this.uniqueFields = new HashSet<FieldSet>(); } this.uniqueFields.addAll(uniqueFieldSets); } } }
CompilerHints defaults = new CompilerHints(); String size = hints.getOutputSize() == defaults.getOutputSize() ? "(none)" : String.valueOf(hints.getOutputSize()); String card = hints.getOutputCardinality() == defaults.getOutputCardinality() ? "(none)" : String.valueOf(hints.getOutputCardinality()); String width = hints.getAvgOutputRecordSize() == defaults.getAvgOutputRecordSize() ? "(none)" : String.valueOf(hints.getAvgOutputRecordSize()); String filter = hints.getFilterFactor() == defaults.getFilterFactor() ? "(none)" : String.valueOf(hints.getFilterFactor());
/** * Creates a new builder for the given configuration. * * @param contract The contract from which the the compiler hints are used. * If contract is null, new compiler hints are generated. * @param config The configuration into which the parameters will be written. */ protected AbstractConfigBuilder(Operator<?> contract, Configuration config) { super(config); if (contract != null) { this.hints = new RecordFormatCompilerHints(contract.getCompilerHints()); // initialize with 2 bytes length for the header (its actually 3, but one is skipped on the first field this.hints.addWidthRecordFormat(2); } else { this.hints = new RecordFormatCompilerHints(new CompilerHints()); } }
filterO1.getCompilerHints().setFilterFactor(0.05f);
CompilerHints defaults = new CompilerHints(); String size = hints.getOutputSize() == defaults.getOutputSize() ? "(none)" : String.valueOf(hints.getOutputSize()); String card = hints.getOutputCardinality() == defaults.getOutputCardinality() ? "(none)" : String.valueOf(hints.getOutputCardinality()); String width = hints.getAvgOutputRecordSize() == defaults.getAvgOutputRecordSize() ? "(none)" : String.valueOf(hints.getAvgOutputRecordSize()); String filter = hints.getFilterFactor() == defaults.getFilterFactor() ? "(none)" : String.valueOf(hints.getFilterFactor());