@Override public void routeDataMovementEventToDestination(DataMovementEvent event, int sourceTaskIndex, int sourceOutputIndex, Map<Integer, List<Integer>> destinationTaskAndInputIndices) { int sourceIndex = event.getSourceIndex(); int destinationTaskIndex = sourceIndex/basePartitionRange; int partitionRange = 1; if(destinationTaskIndex < numDestinationTasks-1) { partitionRange = basePartitionRange; } else { partitionRange = remainderRangeForLastShuffler; } // all inputs from a source task are next to each other in original order int targetIndex = sourceTaskIndex * partitionRange + sourceIndex % partitionRange; destinationTaskAndInputIndices.put( destinationTaskIndex, Collections.singletonList(targetIndex)); }
if (isDataMovementEvent) { DataMovementEvent dmEvent = (DataMovementEvent) event; e = DataMovementEvent.create(dmEvent.getSourceIndex(), inputIndex, dmEvent.getVersion(), dmEvent.getUserPayload()); } else {
private void processDataMovementEvent(DataMovementEvent dmEvent, DataMovementEventPayloadProto shufflePayload, BitSet emptyPartitionsBitSet) throws IOException { int partitionId = dmEvent.getSourceIndex(); CompositeInputAttemptIdentifier srcAttemptIdentifier = constructInputAttemptIdentifier(dmEvent.getTargetIndex(), 1, dmEvent.getVersion(), shufflePayload); if (LOG.isDebugEnabled()) { LOG.debug("DME srcIdx: " + partitionId + ", targetIdx: " + dmEvent.getTargetIndex() + ", attemptNum: " + dmEvent.getVersion() + ", payload: " + ShuffleUtils.stringify(shufflePayload)); } if (shufflePayload.hasEmptyPartitions()) { try { if (emptyPartitionsBitSet.get(partitionId)) { if (LOG.isDebugEnabled()) { LOG.debug( "Source partition: " + partitionId + " did not generate any data. SrcAttempt: [" + srcAttemptIdentifier + "]. Not fetching."); } numDmeEventsNoData.getAndIncrement(); scheduler.copySucceeded(srcAttemptIdentifier.expand(0), null, 0, 0, 0, null, true); return; } } catch (IOException e) { throw new TezUncheckedException("Unable to set the empty partition to succeeded", e); } } scheduler.addKnownMapOutput(StringInterner.weakIntern(shufflePayload.getHost()), shufflePayload.getPort(), partitionId, srcAttemptIdentifier); }
private void processDataMovementEvent(DataMovementEvent dme, DataMovementEventPayloadProto shufflePayload, BitSet emptyPartitionsBitSet) throws IOException { int srcIndex = dme.getSourceIndex(); if (LOG.isDebugEnabled()) { LOG.debug("DME srcIdx: " + srcIndex + ", targetIndex: " + dme.getTargetIndex() + ", attemptNum: " + dme.getVersion() + ", payload: " + ShuffleUtils .stringify(shufflePayload)); } if (shufflePayload.hasEmptyPartitions()) { if (emptyPartitionsBitSet.get(srcIndex)) { CompositeInputAttemptIdentifier srcAttemptIdentifier = constructInputAttemptIdentifier(dme.getTargetIndex(), 1, dme.getVersion(), shufflePayload, false); if (LOG.isDebugEnabled()) { LOG.debug("Source partition: " + srcIndex + " did not generate any data. SrcAttempt: [" + srcAttemptIdentifier + "]. Not fetching."); } numDmeEventsNoData.getAndIncrement(); shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier.expand(0)); return; } } CompositeInputAttemptIdentifier srcAttemptIdentifier = constructInputAttemptIdentifier(dme.getTargetIndex(), 1, dme.getVersion(), shufflePayload, (useSharedInputs && srcIndex == 0)); shuffleManager.addKnownInput(shufflePayload.getHost(), shufflePayload.getPort(), srcAttemptIdentifier, srcIndex); }
public static EventProtos.DataMovementEventProto convertDataMovementEventToProto( DataMovementEvent event) { EventProtos.DataMovementEventProto.Builder builder = EventProtos.DataMovementEventProto.newBuilder(); builder.setSourceIndex(event.getSourceIndex()). setTargetIndex(event.getTargetIndex()).setVersion(event.getVersion()); if (event.getUserPayload() != null) { builder.setUserPayload(ByteString.copyFrom(event.getUserPayload())); } return builder.build(); }
DataMovementEvent dmEvent = (DataMovementEvent) tezEvent.getEvent(); edgeManager.routeDataMovementEventToDestination(dmEvent, srcTaskIndex, dmEvent.getSourceIndex(), destTaskAndInputIndices); } else {
} else { routeMeta = edgeManagerOnDemand.routeDataMovementEventToDestination(srcTaskIndex, dmEvent.getSourceIndex(), taskIndex); numEventsDone = 0;