public static boolean isFittingIntoMemory(Double maxSize, RelNode input, int buckets) { Double currentMemory = RelMetadataQuery.instance().cumulativeMemoryWithinPhase(input); if (currentMemory != null) { if(currentMemory / buckets > maxSize) { return false; } return true; } return false; }
assertThat(mq.cumulativeMemoryWithinPhase(aggregate), nullValue()); assertThat(mq.cumulativeMemoryWithinPhaseSplit(aggregate),
public static boolean isFittingIntoMemory(Double maxSize, RelNode input, int buckets) { final RelMetadataQuery mq = input.getCluster().getMetadataQuery(); Double currentMemory = mq.cumulativeMemoryWithinPhase(input); if (currentMemory != null) { if(currentMemory / buckets > maxSize) { return false; } return true; } return false; }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { RelMetadataQuery mq = RelMetadataQuery.instance(); // TODO: Split count is not same as no of buckets JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezSMBJoinAlgorithm.INSTANCE); final Double memoryWithinPhase = mq.cumulativeMemoryWithinPhase(join); final Integer splitCount = mq.splitCount(join); join.setJoinAlgorithm(oldAlgo); if (memoryWithinPhase == null || splitCount == null) { return null; } return memoryWithinPhase / splitCount; }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezCommonJoinAlgorithm.INSTANCE); final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double memoryWithinPhase = mq.cumulativeMemoryWithinPhase(join); final Integer splitCount = mq.splitCount(join); join.setJoinAlgorithm(oldAlgo); if (memoryWithinPhase == null || splitCount == null) { return null; } return memoryWithinPhase / splitCount; }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { // Check streaming side RelNode inMemoryInput; if (join.getStreamingSide() == MapJoinStreamingRelation.LEFT_RELATION) { inMemoryInput = join.getRight(); } else if (join.getStreamingSide() == MapJoinStreamingRelation.RIGHT_RELATION) { inMemoryInput = join.getLeft(); } else { return null; } // If simple map join, the whole relation goes in memory return RelMetadataQuery.instance().cumulativeMemoryWithinPhase(inMemoryInput); }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); // TODO: Split count is not same as no of buckets JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezSMBJoinAlgorithm.INSTANCE); final Double memoryWithinPhase = mq.cumulativeMemoryWithinPhase(join); final Integer splitCount = mq.splitCount(join); join.setJoinAlgorithm(oldAlgo); if (memoryWithinPhase == null || splitCount == null) { return null; } return memoryWithinPhase / splitCount; }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { // Check streaming side RelNode inMemoryInput; if (join.getStreamingSide() == MapJoinStreamingRelation.LEFT_RELATION) { inMemoryInput = join.getRight(); } else if (join.getStreamingSide() == MapJoinStreamingRelation.RIGHT_RELATION) { inMemoryInput = join.getLeft(); } else { return null; } // If simple map join, the whole relation goes in memory final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); return mq.cumulativeMemoryWithinPhase(inMemoryInput); }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezCommonJoinAlgorithm.INSTANCE); final Double memoryWithinPhase = RelMetadataQuery.instance().cumulativeMemoryWithinPhase(join); final Integer splitCount = RelMetadataQuery.instance().splitCount(join); join.setJoinAlgorithm(oldAlgo); if (memoryWithinPhase == null || splitCount == null) { return null; } return memoryWithinPhase / splitCount; }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { // Check streaming side RelNode inMemoryInput; if (join.getStreamingSide() == MapJoinStreamingRelation.LEFT_RELATION) { inMemoryInput = join.getRight(); } else if (join.getStreamingSide() == MapJoinStreamingRelation.RIGHT_RELATION) { inMemoryInput = join.getLeft(); } else { return null; } // If bucket map join, only a split goes in memory final RelMetadataQuery mq = join.getCluster().getMetadataQuery(); final Double memoryInput = mq.cumulativeMemoryWithinPhase(inMemoryInput); final Integer splitCount = mq.splitCount(inMemoryInput); if (memoryInput == null || splitCount == null) { return null; } return memoryInput / splitCount; }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { // Check streaming side RelNode inMemoryInput; if (join.getStreamingSide() == MapJoinStreamingRelation.LEFT_RELATION) { inMemoryInput = join.getRight(); } else if (join.getStreamingSide() == MapJoinStreamingRelation.RIGHT_RELATION) { inMemoryInput = join.getLeft(); } else { return null; } // If bucket map join, only a split goes in memory final Double memoryInput = RelMetadataQuery.instance().cumulativeMemoryWithinPhase(inMemoryInput); final Integer splitCount = RelMetadataQuery.instance().splitCount(inMemoryInput); if (memoryInput == null || splitCount == null) { return null; } return memoryInput / splitCount; }
public static boolean isFittingIntoMemory(Double maxSize, RelNode input, int buckets) { Double currentMemory = RelMetadataQuery.cumulativeMemoryWithinPhase(input); if (currentMemory != null) { if(currentMemory / buckets > maxSize) { return false; } return true; } return false; }
/** Catch-all implementation for * {@link BuiltInMetadata.Memory#cumulativeMemoryWithinPhaseSplit()}, * invoked using reflection. * * @see org.apache.calcite.rel.metadata.RelMetadataQuery#cumulativeMemoryWithinPhaseSplit */ public Double cumulativeMemoryWithinPhaseSplit(RelNode rel, RelMetadataQuery mq) { final Double memoryWithinPhase = mq.cumulativeMemoryWithinPhase(rel); final Integer splitCount = mq.splitCount(rel); if (memoryWithinPhase == null || splitCount == null) { return null; } return memoryWithinPhase / splitCount; } }
/** Catch-all implementation for * {@link BuiltInMetadata.Memory#cumulativeMemoryWithinPhaseSplit()}, * invoked using reflection. * * @see org.apache.calcite.rel.metadata.RelMetadataQuery#cumulativeMemoryWithinPhaseSplit */ public Double cumulativeMemoryWithinPhaseSplit(RelNode rel, RelMetadataQuery mq) { final Double memoryWithinPhase = mq.cumulativeMemoryWithinPhase(rel); final Integer splitCount = mq.splitCount(rel); if (memoryWithinPhase == null || splitCount == null) { return null; } return memoryWithinPhase / splitCount; } }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { // TODO: Split count is not same as no of buckets JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezSMBJoinAlgorithm.INSTANCE); final Double memoryWithinPhase = RelMetadataQuery.cumulativeMemoryWithinPhase(join); final Integer splitCount = RelMetadataQuery.splitCount(join); join.setJoinAlgorithm(oldAlgo); if (memoryWithinPhase == null || splitCount == null) { return null; } return memoryWithinPhase / splitCount; }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { JoinAlgorithm oldAlgo = join.getJoinAlgorithm(); join.setJoinAlgorithm(TezCommonJoinAlgorithm.INSTANCE); final Double memoryWithinPhase = RelMetadataQuery.cumulativeMemoryWithinPhase(join); final Integer splitCount = RelMetadataQuery.splitCount(join); join.setJoinAlgorithm(oldAlgo); if (memoryWithinPhase == null || splitCount == null) { return null; } return memoryWithinPhase / splitCount; }
/** Catch-all implementation for * {@link BuiltInMetadata.Memory#cumulativeMemoryWithinPhase()}, * invoked using reflection. * * @see org.apache.calcite.rel.metadata.RelMetadataQuery#memory */ public Double cumulativeMemoryWithinPhase(RelNode rel, RelMetadataQuery mq) { Double nullable = mq.memory(rel); if (nullable == null) { return null; } Boolean isPhaseTransition = mq.isPhaseTransition(rel); if (isPhaseTransition == null) { return null; } double d = nullable; if (!isPhaseTransition) { for (RelNode input : rel.getInputs()) { nullable = mq.cumulativeMemoryWithinPhase(input); if (nullable == null) { return null; } d += nullable; } } return d; }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { // Check streaming side RelNode inMemoryInput; if (join.getStreamingSide() == MapJoinStreamingRelation.LEFT_RELATION) { inMemoryInput = join.getRight(); } else if (join.getStreamingSide() == MapJoinStreamingRelation.RIGHT_RELATION) { inMemoryInput = join.getLeft(); } else { return null; } // If simple map join, the whole relation goes in memory return RelMetadataQuery.cumulativeMemoryWithinPhase(inMemoryInput); }
/** Catch-all implementation for * {@link BuiltInMetadata.Memory#cumulativeMemoryWithinPhase()}, * invoked using reflection. * * @see org.apache.calcite.rel.metadata.RelMetadataQuery#memory */ public Double cumulativeMemoryWithinPhase(RelNode rel, RelMetadataQuery mq) { Double nullable = mq.memory(rel); if (nullable == null) { return null; } Boolean isPhaseTransition = mq.isPhaseTransition(rel); if (isPhaseTransition == null) { return null; } double d = nullable; if (!isPhaseTransition) { for (RelNode input : rel.getInputs()) { nullable = mq.cumulativeMemoryWithinPhase(input); if (nullable == null) { return null; } d += nullable; } } return d; }
@Override public Double getCumulativeMemoryWithinPhaseSplit(HiveJoin join) { // Check streaming side RelNode inMemoryInput; if (join.getStreamingSide() == MapJoinStreamingRelation.LEFT_RELATION) { inMemoryInput = join.getRight(); } else if (join.getStreamingSide() == MapJoinStreamingRelation.RIGHT_RELATION) { inMemoryInput = join.getLeft(); } else { return null; } // If bucket map join, only a split goes in memory final Double memoryInput = RelMetadataQuery.cumulativeMemoryWithinPhase(inMemoryInput); final Integer splitCount = RelMetadataQuery.splitCount(inMemoryInput); if (memoryInput == null || splitCount == null) { return null; } return memoryInput / splitCount; }