private void initializeSourceForTag(ReduceWork redWork, int tag, ObjectInspector[] ois, ReduceRecordSource[] sources, TableDesc valueTableDesc, String inputName) throws Exception { reducer = redWork.getReducer(); reducer.getParentOperators().clear(); reducer.setParentOperators(null); // clear out any parents as reducer is the root TableDesc keyTableDesc = redWork.getKeyDesc(); Reader reader = inputs.get(inputName).getReader(); sources[tag] = new ReduceRecordSource(); // Only the big table input source should be vectorized (if applicable) // Note this behavior may have to change if we ever implement a vectorized merge join boolean vectorizedRecordSource = (tag == bigTablePosition) && redWork.getVectorMode(); sources[tag].init(jconf, redWork.getReducer(), vectorizedRecordSource, keyTableDesc, valueTableDesc, reader, tag == bigTablePosition, (byte) tag, redWork.getVectorizedRowBatchCtx(), redWork.getVectorizedVertexNum(), redWork.getVectorizedTestingReducerBatchSize()); ois[tag] = sources[tag].getObjectInspector(); }
private boolean pushRecordVector() { try { if (!reader.next()) { return false; } BytesWritable keyWritable = (BytesWritable) reader.getCurrentKey(); valueWritables = reader.getCurrentValues(); processVectorGroup(keyWritable, valueWritables, tag); return true; } catch (Throwable e) { abort = true; if (e instanceof OutOfMemoryError) { // Don't create a new object if we are already out of memory throw (OutOfMemoryError) e; } else { l4j.error(StringUtils.stringifyException(e)); throw new RuntimeException(e); } } }
@Override void run() throws Exception { for (Entry<String, LogicalOutput> outputEntry : outputs.entrySet()) { l4j.info("Starting Output: " + outputEntry.getKey()); if (!isAborted()) { outputEntry.getValue().start(); ((TezKVOutputCollector) outMap.get(outputEntry.getKey())).initialize(); } } // run the operator pipeline startAbortChecks(); while (sources[bigTablePosition].pushRecord()) { addRowAndMaybeCheckAbort(); } }
return pushRecordVector();
if (isAborted()) { for (ReduceRecordSource rs: sources) { if (!rs.close()) { setAborted(false); // Preserving the old logic. Hmm... break;
((ReduceRecordSource) source).setFlushLastRecord(true);
return pushRecordVector();
if (isAborted()) { for (ReduceRecordSource rs: sources) { if (!rs.close()) { setAborted(false); // Preserving the old logic. Hmm... break;
private void initializeSourceForTag(ReduceWork redWork, int tag, ObjectInspector[] ois, ReduceRecordSource[] sources, TableDesc valueTableDesc, String inputName) throws Exception { reducer = redWork.getReducer(); reducer.getParentOperators().clear(); reducer.setParentOperators(null); // clear out any parents as reducer is the root TableDesc keyTableDesc = redWork.getKeyDesc(); Reader reader = inputs.get(inputName).getReader(); sources[tag] = new ReduceRecordSource(); // Only the big table input source should be vectorized (if applicable) // Note this behavior may have to change if we ever implement a vectorized merge join boolean vectorizedRecordSource = (tag == bigTablePosition) && redWork.getVectorMode(); sources[tag].init(jconf, redWork.getReducer(), vectorizedRecordSource, keyTableDesc, valueTableDesc, reader, tag == bigTablePosition, (byte) tag, redWork.getVectorizedRowBatchCtx(), redWork.getVectorizedVertexNum()); ois[tag] = sources[tag].getObjectInspector(); }
@Override void run() throws Exception { for (Entry<String, LogicalOutput> outputEntry : outputs.entrySet()) { l4j.info("Starting Output: " + outputEntry.getKey()); if (!isAborted()) { outputEntry.getValue().start(); ((TezKVOutputCollector) outMap.get(outputEntry.getKey())).initialize(); } } // run the operator pipeline startAbortChecks(); while (sources[bigTablePosition].pushRecord()) { addRowAndMaybeCheckAbort(); } }
processVectorGroup(keyWritable, valueWritables, tag); return true; } catch (Throwable e) {
abort = abort && rs.close();
private void initializeSourceForTag(ReduceWork redWork, int tag, ObjectInspector[] ois, ReduceRecordSource[] sources, TableDesc valueTableDesc, String inputName) throws Exception { reducer = redWork.getReducer(); reducer.getParentOperators().clear(); reducer.setParentOperators(null); // clear out any parents as reducer is the root TableDesc keyTableDesc = redWork.getKeyDesc(); KeyValuesReader reader = (KeyValuesReader) inputs.get(inputName).getReader(); sources[tag] = new ReduceRecordSource(); sources[tag].init(jconf, redWork.getReducer(), redWork.getVectorMode(), keyTableDesc, valueTableDesc, reader, tag == bigTablePosition, (byte) tag, redWork.getVectorScratchColumnTypeMap()); ois[tag] = sources[tag].getObjectInspector(); }
@Override void run() throws Exception { for (Entry<String, LogicalOutput> outputEntry : outputs.entrySet()) { l4j.info("Starting Output: " + outputEntry.getKey()); outputEntry.getValue().start(); ((TezKVOutputCollector) outMap.get(outputEntry.getKey())).initialize(); } // run the operator pipeline while (sources[bigTablePosition].pushRecord()) { } }
processVectorGroup(keyWritable, valueWritables, tag); return true; } catch (Throwable e) {