public Integer splitCountRepartition(RelNode rel, RelMetadataQuery mq) { // We repartition: new number of splits final Double averageRowSize = mq.getAverageRowSize(rel); final Double rowCount = mq.getRowCount(rel); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }
public Integer splitCountRepartition(RelNode rel, RelMetadataQuery mq) { // We repartition: new number of splits final Double averageRowSize = mq.getAverageRowSize(rel); final Double rowCount = mq.getRowCount(rel); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }
@Override public RelOptCost getScanCost(HiveTableScan ts, RelMetadataQuery mq) { return algoUtils.computeScanCost(mq.getRowCount(ts), mq.getAverageRowSize(ts)); }
public Double memory(HiveAggregate aggregate, RelMetadataQuery mq) { final Double avgRowSize = mq.getAverageRowSize(aggregate.getInput()); final Double rowCount = mq.getRowCount(aggregate.getInput()); if (avgRowSize == null || rowCount == null) { return null; } return avgRowSize * rowCount; }
public Double memory(HiveAggregate aggregate, RelMetadataQuery mq) { final Double avgRowSize = mq.getAverageRowSize(aggregate.getInput()); final Double rowCount = mq.getRowCount(aggregate.getInput()); if (avgRowSize == null || rowCount == null) { return null; } return avgRowSize * rowCount; }
@Override public RelOptCost getScanCost(HiveTableScan ts) { return algoUtils.computeScanCost(ts.getRows(), RelMetadataQuery.instance().getAverageRowSize(ts)); }
public Double memory(HiveSortLimit sort, RelMetadataQuery mq) { if (sort.getCollation() != RelCollations.EMPTY) { // It sorts final Double avgRowSize = mq.getAverageRowSize(sort.getInput()); final Double rowCount = mq.getRowCount(sort.getInput()); if (avgRowSize == null || rowCount == null) { return null; } return avgRowSize * rowCount; } // It does not sort, memory footprint is zero return 0.0; }
public Double memory(HiveSortLimit sort, RelMetadataQuery mq) { if (sort.getCollation() != RelCollations.EMPTY) { // It sorts final Double avgRowSize = mq.getAverageRowSize(sort.getInput()); final Double rowCount = mq.getRowCount(sort.getInput()); if (avgRowSize == null || rowCount == null) { return null; } return avgRowSize * rowCount; } // It does not sort, memory footprint is zero return 0.0; }
final Double leftRAverageSize = mq.getAverageRowSize(join.getLeft()); final Double rightRAverageSize = mq.getAverageRowSize(join.getRight()); if (leftRAverageSize == null || rightRAverageSize == null) { return null;
public static Double getJoinMemory(HiveJoin join, MapJoinStreamingRelation streamingSide) { Double memory = 0.0; RelMetadataQuery mq = RelMetadataQuery.instance(); if (streamingSide == MapJoinStreamingRelation.NONE || streamingSide == MapJoinStreamingRelation.RIGHT_RELATION) { // Left side final Double leftAvgRowSize = mq.getAverageRowSize(join.getLeft()); final Double leftRowCount = mq.getRowCount(join.getLeft()); if (leftAvgRowSize == null || leftRowCount == null) { return null; } memory += leftAvgRowSize * leftRowCount; } if (streamingSide == MapJoinStreamingRelation.NONE || streamingSide == MapJoinStreamingRelation.LEFT_RELATION) { // Right side final Double rightAvgRowSize = mq.getAverageRowSize(join.getRight()); final Double rightRowCount = mq.getRowCount(join.getRight()); if (rightAvgRowSize == null || rightRowCount == null) { return null; } memory += rightAvgRowSize * rightRowCount; } return memory; }
public static Double getJoinMemory(HiveJoin join, MapJoinStreamingRelation streamingSide) { Double memory = 0.0; final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); if (streamingSide == MapJoinStreamingRelation.NONE || streamingSide == MapJoinStreamingRelation.RIGHT_RELATION) { // Left side final Double leftAvgRowSize = mq.getAverageRowSize(join.getLeft()); final Double leftRowCount = mq.getRowCount(join.getLeft()); if (leftAvgRowSize == null || leftRowCount == null) { return null; } memory += leftAvgRowSize * leftRowCount; } if (streamingSide == MapJoinStreamingRelation.NONE || streamingSide == MapJoinStreamingRelation.LEFT_RELATION) { // Right side final Double rightAvgRowSize = mq.getAverageRowSize(join.getRight()); final Double rightRowCount = mq.getRowCount(join.getRight()); if (rightAvgRowSize == null || rightRowCount == null) { return null; } memory += rightAvgRowSize * rightRowCount; } return memory; }
public static Integer getSplitCountWithRepartition(HiveJoin join) { final Double maxSplitSize = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxSplitSize(); // We repartition: new number of splits final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double averageRowSize = mq.getAverageRowSize(join); final Double rowCount = mq.getRowCount(join); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }
public static Integer getSplitCountWithRepartition(HiveJoin join) { final Double maxSplitSize = join.getCluster().getPlanner().getContext(). unwrap(HiveAlgorithmsConf.class).getMaxSplitSize(); // We repartition: new number of splits RelMetadataQuery mq = RelMetadataQuery.instance(); final Double averageRowSize = mq.getAverageRowSize(join); final Double rowCount = mq.getRowCount(join); if (averageRowSize == null || rowCount == null) { return null; } final Double totalSize = averageRowSize * rowCount; final Double splitCount = totalSize / maxSplitSize; return splitCount.intValue(); }
final Double leftRAverageSize = mq.getAverageRowSize(join.getLeft()); final Double rightRAverageSize = mq.getAverageRowSize(join.getRight()); if (leftRAverageSize == null || rightRAverageSize == null) { return null;
final Double leftRAverageSize = mq.getAverageRowSize(join.getLeft()); final Double rightRAverageSize = mq.getAverageRowSize(join.getRight()); if (leftRAverageSize == null || rightRAverageSize == null) { return null;
final Double leftRAverageSize = mq.getAverageRowSize(join.getLeft()); final Double rightRAverageSize = mq.getAverageRowSize(join.getRight()); if (leftRAverageSize == null || rightRAverageSize == null) { return null;
final Double leftRAverageSize = mq.getAverageRowSize(join.getLeft()); final Double rightRAverageSize = mq.getAverageRowSize(join.getRight()); if (leftRAverageSize == null || rightRAverageSize == null) { return null;
@Override public RelOptCost getAggregateCost(HiveAggregate aggregate) { if (aggregate.isBucketedInput()) { return HiveCost.FACTORY.makeZeroCost(); } else { RelMetadataQuery mq = RelMetadataQuery.instance(); // 1. Sum of input cardinalities final Double rCount = mq.getRowCount(aggregate.getInput()); if (rCount == null) { return null; } // 2. CPU cost = sorting cost final double cpuCost = algoUtils.computeSortCPUCost(rCount); // 3. IO cost = cost of writing intermediary results to local FS + // cost of reading from local FS for transferring to GBy + // cost of transferring map outputs to GBy operator final Double rAverageSize = mq.getAverageRowSize(aggregate.getInput()); if (rAverageSize == null) { return null; } final double ioCost = algoUtils.computeSortIOCost(new Pair<Double,Double>(rCount,rAverageSize)); // 4. Result return HiveCost.FACTORY.makeCost(rCount, cpuCost, ioCost); } }
@Override public RelOptCost getAggregateCost(HiveAggregate aggregate) { if (aggregate.isBucketedInput()) { return HiveCost.FACTORY.makeZeroCost(); } else { final RelMetadataQuery mq = aggregate.getCluster().getMetadataQuery(); // 1. Sum of input cardinalities final Double rCount = mq.getRowCount(aggregate.getInput()); if (rCount == null) { return null; } // 2. CPU cost = sorting cost final double cpuCost = algoUtils.computeSortCPUCost(rCount); // 3. IO cost = cost of writing intermediary results to local FS + // cost of reading from local FS for transferring to GBy + // cost of transferring map outputs to GBy operator final Double rAverageSize = mq.getAverageRowSize(aggregate.getInput()); if (rAverageSize == null) { return null; } final double ioCost = algoUtils.computeSortIOCost(new Pair<Double,Double>(rCount,rAverageSize)); // 4. Result return HiveCost.FACTORY.makeCost(rCount, cpuCost, ioCost); } }
final Double leftRAverageSize = mq.getAverageRowSize(join.getLeft()); final Double rightRAverageSize = mq.getAverageRowSize(join.getRight()); if (leftRAverageSize == null || rightRAverageSize == null) { return null;