public static EventProtos.VertexManagerEventProto convertVertexManagerEventToProto( VertexManagerEvent event) { EventProtos.VertexManagerEventProto.Builder vmBuilder = VertexManagerEventProto.newBuilder(); vmBuilder.setTargetVertexName(event.getTargetVertexName()); if (event.getUserPayload() != null) { vmBuilder.setUserPayload(ByteString.copyFrom(event.getUserPayload())); } return vmBuilder.build(); }
/** * Create a new VertexManagerEvent * @param vertexName * @param userPayload This should not be modified since a reference is kept */ public static VertexManagerEvent create(String vertexName, ByteBuffer userPayload) { return new VertexManagerEvent(vertexName, userPayload); }
public static VertexManagerEvent convertVertexManagerEventFromProto( EventProtos.VertexManagerEventProto vmProto) { return VertexManagerEvent.create(vmProto.getTargetVertexName(), vmProto.hasUserPayload() ? vmProto.getUserPayload().asReadOnlyByteBuffer() : null); }
@Override public synchronized void onVertexManagerEventReceived(VertexManagerEvent vmEvent) throws IOException { /* vmEvent after reconfigure doesn't matter */ if (vertexReconfigured) { return; } if (vmEvent.getUserPayload() != null) { String srcVertex = vmEvent.getProducerAttemptIdentifier().getTaskIdentifier().getVertexIdentifier().getName(); SrcVertex srcV = srcVerticesByName.get(srcVertex); // vmEvent from non-cp vertex doesn't matter if (srcV == null) { return; } VertexManagerEventPayloadProto proto = VertexManagerEventPayloadProto.parseFrom(ByteString.copyFrom(vmEvent.getUserPayload())); srcV.numRecord += proto.getNumRecord(); srcV.taskWithVMEvent.add( vmEvent.getProducerAttemptIdentifier().getTaskIdentifier().getIdentifier()); } tryScheduleTasks(); }
if (vmEvent.getUserPayload().limit()==4) { dynamicParallelism = vmEvent.getUserPayload().getInt(); } else { return;
Vertex target = getDAG().getVertex(vmEvent.getTargetVertexName()); Preconditions.checkArgument(target != null, "Event sent to unkown vertex: " + vmEvent.getTargetVertexName()); TezTaskAttemptID srcTaId = sourceMeta.getTaskAttemptID(); if (srcTaId.getTaskID().getVertexID().equals(vertexId)) { vmEvent.setProducerAttemptIdentifier( getTaskAttemptIdentifier(dag.getName(), getName(), srcTaId));
VertexManagerEvent vmEvent = VertexManagerEvent.create( sortingVertex, ByteBuffer.wrap(Ints.toByteArray(parallelism))); List<Event> events = Lists.newArrayListWithCapacity(1);
private void handleVertexManagerEvent(VertexManagerEvent vmEvent) { TaskIdentifier producerTask = vmEvent.getProducerAttemptIdentifier().getTaskIdentifier(); if (!taskWithVmEvents.add(producerTask)) { LOG.info("Ignoring vertex manager event from: {}", producerTask); if (vmEvent.getUserPayload() != null) { ByteString.copyFrom(vmEvent.getUserPayload())); } catch (InvalidProtocolBufferException e) { throw new TezUncheckedException(e); + " vertex numEventsReceived: {} vertex output size: {}" + " total numEventsReceived: {} total output size: {}", vmEvent.getProducerAttemptIdentifier(), sourceTaskOutputSize, srcInfo.numVMEventsReceived, srcInfo.outputSize, numVertexManagerEventsReceived, completedSourceTasksOutputSize);
ShuffleUserPayloads.VertexManagerEventPayloadProto.newBuilder(); vmBuilder.setOutputSize(0); VertexManagerEvent vmEvent = VertexManagerEvent.create( context.getDestinationVertexName(), vmBuilder.build().toByteString().asReadOnlyByteBuffer());
VertexManagerEvent vmEvent = VertexManagerEvent.create( context.getDestinationVertexName(), vmBuilder.build().toByteString().asReadOnlyByteBuffer());