private void adjustAutoParallelism(BaseWork work) { if (minReducersPerExec <= 0 || !(work instanceof ReduceWork)) return; ReduceWork reduceWork = (ReduceWork)work; if (reduceWork.isAutoReduceParallelism() == false && reduceWork.isUniformDistribution() == false) { return; // Not based on ARP and cannot assume uniform distribution, bail. if (reduceWork.isAutoReduceParallelism()) {
if (rWork.isAutoReduceParallelism()) { edgeProp = new TezEdgeProperty(context.conf, edgeType, true, rWork.isSlowStart(),
private Vertex createVertex(JobConf conf, ReduceWork reduceWork, LocalResource appJarLr, List<LocalResource> additionalLr, FileSystem fs, Path mrScratchDir, Context ctx) throws Exception { // set up operator plan conf.set(Utilities.INPUT_NAME, reduceWork.getName()); Utilities.setReduceWork(conf, reduceWork, mrScratchDir, false); // create the directories FileSinkOperators need Utilities.createTmpDirs(conf, reduceWork); VertexExecutionContext vertexExecutionContext = createVertexExecutionContext(reduceWork); // create the vertex Vertex reducer = Vertex.create(reduceWork.getName(), ProcessorDescriptor.create(ReduceTezProcessor.class.getName()). setUserPayload(TezUtils.createUserPayloadFromConf(conf)), reduceWork.isAutoReduceParallelism() ? reduceWork.getMaxReduceTasks() : reduceWork.getNumReduceTasks(), getContainerResource(conf)); reducer.setTaskEnvironment(getContainerEnvironment(conf, false)); reducer.setExecutionContext(vertexExecutionContext); reducer.setTaskLaunchCmdOpts(getContainerJavaOpts(conf)); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); localResources.put(getBaseName(appJarLr), appJarLr); for (LocalResource lr: additionalLr) { localResources.put(getBaseName(lr), lr); } reducer.addTaskLocalFiles(localResources); return reducer; }
if (rWork.isAutoReduceParallelism()) { edgeProp = new TezEdgeProperty(context.conf, edgeType, true,
private Vertex createVertex(JobConf conf, ReduceWork reduceWork, FileSystem fs, Path mrScratchDir, Context ctx, Map<String, LocalResource> localResources) throws Exception { // set up operator plan conf.set(Utilities.INPUT_NAME, reduceWork.getName()); Utilities.setReduceWork(conf, reduceWork, mrScratchDir, false); // create the directories FileSinkOperators need Utilities.createTmpDirs(conf, reduceWork); VertexExecutionContext vertexExecutionContext = createVertexExecutionContext(reduceWork); // create the vertex Vertex reducer = Vertex.create(reduceWork.getName(), ProcessorDescriptor.create(ReduceTezProcessor.class.getName()). setUserPayload(TezUtils.createUserPayloadFromConf(conf)), reduceWork.isAutoReduceParallelism() ? reduceWork.getMaxReduceTasks() : reduceWork.getNumReduceTasks(), getContainerResource(conf)); reducer.setTaskEnvironment(getContainerEnvironment(conf, false)); reducer.setExecutionContext(vertexExecutionContext); reducer.setTaskLaunchCmdOpts(getContainerJavaOpts(conf)); reducer.addTaskLocalFiles(localResources); return reducer; }
if (reduceWork.isAutoReduceParallelism()) { edgeProp = new TezEdgeProperty(context.conf, edgeType, true, reduceWork.isSlowStart(),
if (reduceWork.isAutoReduceParallelism()) { edgeProp = new TezEdgeProperty(context.conf, edgeType, true,
private Vertex createVertex(JobConf conf, ReduceWork reduceWork, LocalResource appJarLr, List<LocalResource> additionalLr, FileSystem fs, Path mrScratchDir, Context ctx) throws Exception { // set up operator plan conf.set(Utilities.INPUT_NAME, reduceWork.getName()); Utilities.setReduceWork(conf, reduceWork, mrScratchDir, false); // create the directories FileSinkOperators need Utilities.createTmpDirs(conf, reduceWork); // create the vertex Vertex reducer = Vertex.create(reduceWork.getName(), ProcessorDescriptor.create(ReduceTezProcessor.class.getName()). setUserPayload(TezUtils.createUserPayloadFromConf(conf)), reduceWork.isAutoReduceParallelism() ? reduceWork.getMaxReduceTasks() : reduceWork .getNumReduceTasks(), getContainerResource(conf)); reducer.setTaskEnvironment(getContainerEnvironment(conf, false)); reducer.setTaskLaunchCmdOpts(getContainerJavaOpts(conf)); Map<String, LocalResource> localResources = new HashMap<String, LocalResource>(); localResources.put(getBaseName(appJarLr), appJarLr); for (LocalResource lr: additionalLr) { localResources.put(getBaseName(lr), lr); } reducer.addTaskLocalFiles(localResources); return reducer; }
if (rWork.isAutoReduceParallelism()) { edgeProp = new TezEdgeProperty(context.conf, EdgeType.SIMPLE_EDGE, true,
if (reduceWork.isAutoReduceParallelism()) { edgeProp = new TezEdgeProperty(context.conf, EdgeType.SIMPLE_EDGE, true,