@Private public DataMovementEvent expand(int offset) { return DataMovementEvent.create(sourceIndex + offset, targetIndex + offset, version, userPayload); } }
public static EventProtos.DataMovementEventProto convertDataMovementEventToProto( DataMovementEvent event) { EventProtos.DataMovementEventProto.Builder builder = EventProtos.DataMovementEventProto.newBuilder(); builder.setSourceIndex(event.getSourceIndex()). setTargetIndex(event.getTargetIndex()).setVersion(event.getVersion()); if (event.getUserPayload() != null) { builder.setUserPayload(ByteString.copyFrom(event.getUserPayload())); } return builder.build(); }
/** * User Event constructor for {@link Output}s * @param sourceIndex Index to identify the physical edge of the input/output * that generated the event * @param userPayload User Payload of the User Event */ public static DataMovementEvent create(int sourceIndex, ByteBuffer userPayload) { return new DataMovementEvent(sourceIndex, -1, -1, userPayload); }
private void processDataMovementEvent(DataMovementEvent dme, DataMovementEventPayloadProto shufflePayload, BitSet emptyPartitionsBitSet) throws IOException { int srcIndex = dme.getSourceIndex(); if (LOG.isDebugEnabled()) { LOG.debug("DME srcIdx: " + srcIndex + ", targetIndex: " + dme.getTargetIndex() + ", attemptNum: " + dme.getVersion() + ", payload: " + ShuffleUtils .stringify(shufflePayload)); } if (shufflePayload.hasEmptyPartitions()) { if (emptyPartitionsBitSet.get(srcIndex)) { CompositeInputAttemptIdentifier srcAttemptIdentifier = constructInputAttemptIdentifier(dme.getTargetIndex(), 1, dme.getVersion(), shufflePayload, false); if (LOG.isDebugEnabled()) { LOG.debug("Source partition: " + srcIndex + " did not generate any data. SrcAttempt: [" + srcAttemptIdentifier + "]. Not fetching."); } numDmeEventsNoData.getAndIncrement(); shuffleManager.addCompletedInputWithNoData(srcAttemptIdentifier.expand(0)); return; } } CompositeInputAttemptIdentifier srcAttemptIdentifier = constructInputAttemptIdentifier(dme.getTargetIndex(), 1, dme.getVersion(), shufflePayload, (useSharedInputs && srcIndex == 0)); shuffleManager.addKnownInput(shufflePayload.getHost(), shufflePayload.getPort(), srcAttemptIdentifier, srcIndex); }
if (isDataMovementEvent) { DataMovementEvent dmEvent = (DataMovementEvent) event; e = DataMovementEvent.create(dmEvent.getSourceIndex(), inputIndex, dmEvent.getVersion(), dmEvent.getUserPayload()); } else { InputFailedEvent ifEvent = ((InputFailedEvent) event);
@Override public void routeDataMovementEventToDestination(DataMovementEvent event, int sourceTaskIndex, int sourceOutputIndex, Map<Integer, List<Integer>> destinationTaskAndInputIndices) { int sourceIndex = event.getSourceIndex(); int destinationTaskIndex = sourceIndex/basePartitionRange; int partitionRange = 1; if(destinationTaskIndex < numDestinationTasks-1) { partitionRange = basePartitionRange; } else { partitionRange = remainderRangeForLastShuffler; } // all inputs from a source task are next to each other in original order int targetIndex = sourceTaskIndex * partitionRange + sourceIndex % partitionRange; destinationTaskAndInputIndices.put( destinationTaskIndex, Collections.singletonList(targetIndex)); }
DataMovementEventPayloadProto shufflePayload; try { shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(dmEvent.getUserPayload())); } catch (InvalidProtocolBufferException e) { throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
} else { routeMeta = edgeManagerOnDemand.routeDataMovementEventToDestination(srcTaskIndex, dmEvent.getSourceIndex(), taskIndex); numEventsDone = 0; int[] targetIndices = routeMeta.getTargetIndices(); while (numEventsDone < numEvents && listSize++ < listMaxSize) { DataMovementEvent e = dmEvent.makeCopy(targetIndices[numEventsDone]); numEventsDone++; TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(),
private void processDataMovementEvent(DataMovementEvent dmEvent, DataMovementEventPayloadProto shufflePayload, BitSet emptyPartitionsBitSet) throws IOException { int partitionId = dmEvent.getSourceIndex(); CompositeInputAttemptIdentifier srcAttemptIdentifier = constructInputAttemptIdentifier(dmEvent.getTargetIndex(), 1, dmEvent.getVersion(), shufflePayload); if (LOG.isDebugEnabled()) { LOG.debug("DME srcIdx: " + partitionId + ", targetIdx: " + dmEvent.getTargetIndex() + ", attemptNum: " + dmEvent.getVersion() + ", payload: " + ShuffleUtils.stringify(shufflePayload)); } if (shufflePayload.hasEmptyPartitions()) { try { if (emptyPartitionsBitSet.get(partitionId)) { if (LOG.isDebugEnabled()) { LOG.debug( "Source partition: " + partitionId + " did not generate any data. SrcAttempt: [" + srcAttemptIdentifier + "]. Not fetching."); } numDmeEventsNoData.getAndIncrement(); scheduler.copySucceeded(srcAttemptIdentifier.expand(0), null, 0, 0, 0, null, true); return; } } catch (IOException e) { throw new TezUncheckedException("Unable to set the empty partition to succeeded", e); } } scheduler.addKnownMapOutput(StringInterner.weakIntern(shufflePayload.getHost()), shufflePayload.getPort(), partitionId, srcAttemptIdentifier); }
DataMovementEvent dmEvent = (DataMovementEvent) tezEvent.getEvent(); edgeManager.routeDataMovementEventToDestination(dmEvent, srcTaskIndex, dmEvent.getSourceIndex(), destTaskAndInputIndices); } else {
DataMovementEventPayloadProto shufflePayload; try { shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(dmEvent.getUserPayload())); } catch (InvalidProtocolBufferException e) { throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
public static DataMovementEvent convertDataMovementEventFromProto( EventProtos.DataMovementEventProto proto) { return DataMovementEvent.create(proto.getSourceIndex(), proto.getTargetIndex(), proto.getVersion(), proto.getUserPayload() != null ? proto.getUserPayload().asReadOnlyByteBuffer() : null); } public static EventProtos.CompositeRoutedDataMovementEventProto convertCompositeRoutedDataMovementEventToProto(
@Private /** * Constructor for Processor-generated User Events * @param userPayload */ public static DataMovementEvent create(ByteBuffer userPayload) { return new DataMovementEvent(userPayload); }
eventList.add(cdme); } else { DataMovementEvent dme = DataMovementEvent.create(0, dmePayload); eventList.add(dme);
@Private public static DataMovementEvent create(int sourceIndex, int targetIndex, int version, ByteBuffer userPayload) { return new DataMovementEvent(sourceIndex, targetIndex, version, userPayload); }
/** * Make a routable copy of the {@link DataMovementEvent} by adding a target * input index * * @param targetIndex * The index of the physical input to which this * {@link DataMovementEvent} should be routed * @return Copy of this {@link DataMovementEvent} with the target input index * added to it */ @Private public DataMovementEvent makeCopy(int targetIndex) { return new DataMovementEvent(sourceIndex, targetIndex, version, userPayload); }
/** * Expand the {@link CompositeDataMovementEvent} into a routable * {@link DataMovementEvent} by providing the source output index and the * target input index. * * @param sourceIndex * The index of the physical output represented by the * {@link DataMovementEvent} * @param targetIndex * The index of the physical input to which the given * {@link DataMovementEvent} should be routed. * @return {@link DataMovementEvent} created from the * {@link CompositeDataMovementEvent} with indices specified by the * method parameters */ @Private public DataMovementEvent expand(int sourceIndex, int targetIndex) { return new DataMovementEvent(sourceIndex, targetIndex, version, userPayload); }