this.useMemoryPrediction = context.getOptions().getOption(ExecConstants.HASHAGG_USE_MEMORY_PREDICTION_VALIDATOR); this.phase = hashAggrConfig.getAggPhase(); canSpill = phase.hasTwo(); // single phase can not spill
if ( phase.is1st() && ! tryAvoidCurr) { return currPart; } // 1st phase: just use the current partition int currPartSize = batchHolders[currPart].size(); if ( currPartSize == 1 ) { currPartSize = -1; } // don't pick current if size is 1
allFlushed = true; this.outcome = IterOutcome.NONE; if ( phase.is2nd() && spillSet.getWriteBytes() > 0 ) {
HashTable.PutStatus putStatus = null; long allocatedBeforeHTput = allocator.getAllocatedMemory(); String tryingTo = phase.is1st() ? "early return" : "spill";
if ( numPartitions == 1 && phase.is2nd() ) { // 1st phase can still do early return with 1 partition canSpill = false; logger.warn("Spilling is disabled due to configuration setting of num_partitions to 1"); numPartitions /= 2; if ( numPartitions < 2) { if (phase.is2nd()) { canSpill = false; // 2nd phase needs at least 2 to make progress logger.debug("{} phase. Number of partitions chosen: {}. {} spill", phase.getName(), numPartitions, canSpill ? "Can" : "Cannot");
phase.getName(),estRowWidth,estValuesRowWidth,estMaxBatchSize,allocator.getLimit(),maxColumnWidth);
final AggPrelBase.OperatorPhase phase = popConfig.getAggPhase(); if ( phase.is2nd() && !fallbackEnabled ) { minBatchesNeeded *= 2; // 2nd phase (w/o fallback) needs at least 2 partitions
this.stats.setLongStat(Metric.NUM_PARTITIONS, spilledState.getNumPartitions()); this.stats.setLongStat(Metric.SPILL_CYCLE, spilledState.getCycle()); // Put 0 in case no spill if ( phase.is2nd() ) { this.stats.setLongStat(Metric.SPILLED_PARTITIONS, numSpilled);
@Override public void cleanup() { if ( schema == null ) { return; } // not set up; nothing to clean if ( phase.is2nd() && spillSet.getWriteBytes() > 0 ) {
/** * Generate a detailed error message in case of "Out Of Memory" * @return err msg * @param prefix */ private String getOOMErrorMsg(String prefix) { String errmsg; if (!phase.hasTwo()) { errmsg = "Single Phase Hash Aggregate operator can not spill."; } else if (!canSpill) { // 2nd phase, with only 1 partition errmsg = "Too little memory available to operator to facilitate spilling."; } else { // a bug ? errmsg = prefix + " OOM at " + phase.getName() + " Phase. Partitions: " + spilledState.getNumPartitions() + ". Estimated batch size: " + estMaxBatchSize + ". values size: " + estValuesBatchSize + ". Output alloc size: " + estOutgoingAllocSize; if ( plannedBatches > 0 ) { errmsg += ". Planned batches: " + plannedBatches; } if ( rowsSpilled > 0 ) { errmsg += ". Rows spilled so far: " + rowsSpilled; } } errmsg += " Memory limit: " + allocator.getLimit() + " so far allocated: " + allocator.getAllocatedMemory() + ". "; return errmsg; }