public static EventProtos.CompositeRoutedDataMovementEventProto convertCompositeRoutedDataMovementEventToProto( CompositeRoutedDataMovementEvent event) { EventProtos.CompositeRoutedDataMovementEventProto.Builder builder = EventProtos.CompositeRoutedDataMovementEventProto.newBuilder(); builder.setSourceIndex(event.getSourceIndex()). setTargetIndex(event.getTargetIndex()).setVersion(event.getVersion()).setCount(event.getCount()); if (event.getUserPayload() != null) { builder.setUserPayload(ByteString.copyFrom(event.getUserPayload())); } return builder.build(); }
@Private public static CompositeRoutedDataMovementEvent create(int sourceIndex, int targetIndex, int count, int version, ByteBuffer userPayload) { return new CompositeRoutedDataMovementEvent(sourceIndex, targetIndex, count, version, userPayload); }
DataMovementEventPayloadProto shufflePayload; try { shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(crdme.getUserPayload())); } catch (InvalidProtocolBufferException e) { throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e); numDmeEvents.addAndGet(crdme.getCount()); processCompositeRoutedDataMovementEvent(crdme, shufflePayload, emptyPartitionsBitSet); } else { for (int offset = 0; offset < crdme.getCount(); offset++) { numDmeEvents.incrementAndGet(); processDataMovementEvent(crdme.expand(offset), shufflePayload, emptyPartitionsBitSet);
@Private public CompositeRoutedDataMovementEvent expandRouted(CompositeEventRouteMetadata routeMeta) { return CompositeRoutedDataMovementEvent.create(routeMeta.getSource(), routeMeta.getTarget(), routeMeta.getCount(), version, userPayload); }
DataMovementEventPayloadProto shufflePayload; try { shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(crdme.getUserPayload())); } catch (InvalidProtocolBufferException e) { throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e); numDmeEvents.addAndGet(crdme.getCount()); processCompositeRoutedDataMovementEvent(crdme, shufflePayload, emptyPartitionsBitSet); } else { for (int offset = 0; offset < crdme.getCount(); offset++) { numDmeEvents.incrementAndGet(); processDataMovementEvent(crdme.expand(offset), shufflePayload, emptyPartitionsBitSet);
public static CompositeRoutedDataMovementEvent convertCompositeRoutedDataMovementEventFromProto( EventProtos.CompositeRoutedDataMovementEventProto proto) { return CompositeRoutedDataMovementEvent.create(proto.getSourceIndex(), proto.getTargetIndex(), proto.getCount(), proto.getVersion(), proto.getUserPayload() != null ? proto.getUserPayload().asReadOnlyByteBuffer() : null); }
private void processCompositeRoutedDataMovementEvent(CompositeRoutedDataMovementEvent crdme, DataMovementEventPayloadProto shufflePayload, BitSet emptyPartitionsBitSet) throws IOException { int partitionId = crdme.getSourceIndex(); if (LOG.isDebugEnabled()) { LOG.debug("DME srcIdx: " + partitionId + ", targetIndex: " + crdme.getTargetIndex() + ", count:" + crdme.getCount() + ", attemptNum: " + crdme.getVersion() + ", payload: " + ShuffleUtils .stringify(shufflePayload)); constructInputAttemptIdentifier(crdme.getTargetIndex(), crdme.getCount(), crdme.getVersion(), shufflePayload, false); for (int i = 0; i < crdme.getCount(); i++) { int srcPartitionId = partitionId + i; allPartitionsEmpty &= emptyPartitionsBitSet.get(srcPartitionId); CompositeInputAttemptIdentifier srcAttemptIdentifier = constructInputAttemptIdentifier(crdme.getTargetIndex(), crdme.getCount(), crdme.getVersion(), shufflePayload, (useSharedInputs && partitionId == 0));
private void processCompositeRoutedDataMovementEvent(CompositeRoutedDataMovementEvent crdmEvent, DataMovementEventPayloadProto shufflePayload, BitSet emptyPartitionsBitSet) throws IOException { int partitionId = crdmEvent.getSourceIndex(); CompositeInputAttemptIdentifier compositeInputAttemptIdentifier = constructInputAttemptIdentifier(crdmEvent.getTargetIndex(), crdmEvent.getCount(), crdmEvent.getVersion(), shufflePayload); if (LOG.isDebugEnabled()) { LOG.debug("DME srcIdx: " + partitionId + ", targetIdx: " + crdmEvent.getTargetIndex() + ", count:" + crdmEvent.getCount() + ", attemptNum: " + crdmEvent.getVersion() + ", payload: " + ShuffleUtils.stringify(shufflePayload)); } if (shufflePayload.hasEmptyPartitions()) { boolean allPartitionsEmpty = true; for (int i = 0; i < crdmEvent.getCount(); i++) { int srcPartitionId = partitionId + i; allPartitionsEmpty &= emptyPartitionsBitSet.get(srcPartitionId); if (emptyPartitionsBitSet.get(srcPartitionId)) { InputAttemptIdentifier srcInputAttemptIdentifier = compositeInputAttemptIdentifier.expand(i); if (LOG.isDebugEnabled()) { LOG.debug("Source partition: " + srcPartitionId + " did not generate any data. SrcAttempt: [" + srcInputAttemptIdentifier + "]. Not fetching."); } numDmeEventsNoData.getAndIncrement(); scheduler.copySucceeded(srcInputAttemptIdentifier, null, 0, 0, 0, null, true); } } if (allPartitionsEmpty) { return; } } scheduler.addKnownMapOutput(StringInterner.weakIntern(shufflePayload.getHost()), shufflePayload.getPort(), partitionId, compositeInputAttemptIdentifier); }