/** * Sends out final events for task success. * @param taskAttemptID * @return * @throws IOException * indicates an RPC communication failure. * @throws TezException * indicates an exception somewhere in the AM. */ private boolean taskSucceeded(TezTaskAttemptID taskAttemptID) throws IOException, TezException { // Ensure only one final event is ever sent. if (!finalEventQueued.getAndSet(true)) { TezEvent statusUpdateEvent = new TezEvent( getStatusUpdateEvent(true, true), updateEventMetadata); TezEvent taskCompletedEvent = new TezEvent(new TaskAttemptCompletedEvent(), updateEventMetadata); if (LOG.isDebugEnabled()) { LOG.debug("Invoking OOB heartbeat for successful attempt: {}, isTaskDone={}", taskAttemptID, task.isTaskDone()); } completionListener.fragmentCompleting(fragmentRequestId, SchedulerFragmentCompletingListener.State.SUCCESS); return !heartbeat(Lists.newArrayList(statusUpdateEvent, taskCompletedEvent)).shouldDie; } else { LOG.warn("A final task state event has already been sent. Not sending again"); return askedToDie.get(); } }
@Override public void heartbeat(TezHeartbeatRequest request) { List<TezEvent> inEvents = request.getEvents(); for (TezEvent tezEvent : ListUtils.emptyIfNull(inEvents)) { EventType eventType = tezEvent.getEventType(); try { switch (eventType) { case TASK_ATTEMPT_COMPLETED_EVENT: sendOrQueueEvent(ReaderEvent.doneEvent()); break; case TASK_ATTEMPT_FAILED_EVENT: TaskAttemptFailedEvent taskFailedEvent = (TaskAttemptFailedEvent) tezEvent.getEvent(); sendOrQueueEvent(ReaderEvent.errorEvent(taskFailedEvent.getDiagnostics())); break; case TASK_STATUS_UPDATE_EVENT: // If we want to handle counters break; default: LOG.warn("Unhandled event type " + eventType); break; } } catch (Exception err) { LOG.error("Error during heartbeat responder:", err); } } }
public static TezEvent toTezEvent(NotTezEvent nte) throws InvalidProtocolBufferException { EventMetaData sourceMetaData = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, nte.getVertexName(), "NULL_VERTEX", null); EventMetaData destMetaData = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, nte.getVertexName(), nte.getDestInputName(), null); InputDataInformationEvent event = ProtoConverters.convertRootInputDataInformationEventFromProto( RootInputDataInformationEventProto.parseFrom(nte.getInputEventProtoBytes())); TezEvent tezEvent = new TezEvent(event, sourceMetaData, System.currentTimeMillis()); tezEvent.setDestinationInfo(destMetaData); return tezEvent; } }
public static TezEventProto toProto(TezEvent event) throws IOException { TezEventProto.Builder evtBuilder = TezEventProto.newBuilder(); if (event.getEventType().equals(EventType.COMPOSITE_DATA_MOVEMENT_EVENT)) { evtBuilder.setCompositeDataMovementEvent( ProtoConverters.convertCompositeDataMovementEventToProto( (CompositeDataMovementEvent) event.getEvent())); } else if (event.getEventType().equals(EventType.DATA_MOVEMENT_EVENT)) { evtBuilder.setDataMovementEvent( ProtoConverters.convertDataMovementEventToProto( (DataMovementEvent) event.getEvent())); } else if (event.getEventType().equals(EventType.ROOT_INPUT_INITIALIZER_EVENT)) { evtBuilder.setInputInitializerEvent(ProtoConverters .convertRootInputInitializerEventToProto((InputInitializerEvent) event.getEvent())); } else if (event.getEventType().equals(EventType.VERTEX_MANAGER_EVENT)) { evtBuilder.setVmEvent(ProtoConverters .convertVertexManagerEventToProto((VertexManagerEvent)event.getEvent())); } else if (event.getEventType().equals(EventType.ROOT_INPUT_DATA_INFORMATION_EVENT)) { evtBuilder.setRootInputDataInformationEvent( ProtoConverters.convertRootInputDataInformationEventToProto( (InputDataInformationEvent) event.getEvent())); } else { throw new IOException("Unsupported TezEvent type:" + event.getEventType()); if (event.getSourceInfo() != null) { evtBuilder.setSourceInfo(convertEventMetaDataToProto(event.getSourceInfo())); if (event.getDestinationInfo() != null) { evtBuilder.setDestinationInfo(convertEventMetaDataToProto(event.getDestinationInfo()));
EdgeManagerPluginOnDemand edgeManagerOnDemand = (EdgeManagerPluginOnDemand) edgeManager; int taskIndex = attemptID.getTaskID().getId(); switch (tezEvent.getEventType()) { case COMPOSITE_DATA_MOVEMENT_EVENT: CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent(); CompositeEventRouteMetadata routeMeta = edgeManagerOnDemand .routeCompositeDataMovementEventToDestination(srcTaskIndex, taskIndex); TezEvent tezEventToSend = new TezEvent(edme, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime()); tezEventToSend.setDestinationInfo(destinationMetaInfo); listToAdd.add(tezEventToSend); InputFailedEvent ifEvent = (InputFailedEvent) tezEvent.getEvent(); EventRouteMetadata routeMeta; int numEventsDone; InputFailedEvent e = ifEvent.makeCopy(targetIndices[numEventsDone]); numEventsDone++; TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime()); tezEventToSend.setDestinationInfo(destinationMetaInfo); listToAdd.add(tezEventToSend); DataMovementEvent dmEvent = (DataMovementEvent) tezEvent.getEvent(); EventRouteMetadata routeMeta; int numEventsDone; DataMovementEvent e = dmEvent.makeCopy(targetIndices[numEventsDone]);
private void handleCompositeDataMovementEvent(TezEvent tezEvent) throws AMUserCodeException { CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent(); EventMetaData srcInfo = tezEvent.getSourceInfo(); for (DataMovementEvent dmEvent : compEvent.getEvents()) { TezEvent newEvent = new TezEvent(dmEvent, srcInfo, tezEvent.getEventReceivedTime()); sendTezEventToDestinationTasks(newEvent); } }
tezEvent.setEventReceivedTime(currTime); final EventType eventType = tezEvent.getEventType(); if (eventType == EventType.TASK_STATUS_UPDATE_EVENT) { (TaskStatusUpdateEvent) tezEvent.getEvent()); } else if (eventType == EventType.TASK_ATTEMPT_COMPLETED_EVENT || eventType == EventType.TASK_ATTEMPT_FAILED_EVENT EventMetaData sourceMeta = e.getSourceInfo(); switch (e.getEventType()) { case TASK_ATTEMPT_FAILED_EVENT: case TASK_ATTEMPT_KILLED_EVENT: if (e.getEventType() == EventType.TASK_ATTEMPT_FAILED_EVENT) { TaskAttemptFailedEvent taskFailedEvent = (TaskAttemptFailedEvent) e.getEvent(); sendEvent( new TaskAttemptEventAttemptFailed(sourceMeta.getTaskAttemptID(), errCause)); } else { // Killed TaskAttemptKilledEvent taskKilledEvent = (TaskAttemptKilledEvent) e.getEvent(); sendEvent( new TaskAttemptEventAttemptKilled(sourceMeta.getTaskAttemptID(), default: throw new TezUncheckedException("Unhandled tez event type: " + e.getEventType());
private void sendTaskGeneratedEvents(List<Event> events, EventProducerConsumerType generator, String taskVertexName, String edgeVertexName, TezTaskAttemptID taskAttemptID) { if (events == null || events.isEmpty()) { return; } EventMetaData eventMetaData = new EventMetaData(generator, taskVertexName, edgeVertexName, taskAttemptID); List<TezEvent> tezEvents = new ArrayList<TezEvent>(events.size()); for (Event e : events) { TezEvent te = new TezEvent(e, eventMetaData); tezEvents.add(te); } if (LOG.isDebugEnabled()) { for (TezEvent e : tezEvents) { LOG.debug("Generated event info" + ", eventMetaData=" + eventMetaData.toString() + ", eventType=" + e.getEventType()); } } tezUmbilical.addEvents(tezEvents); }
while (eventIterator.hasNext()) { TezEvent tezEvent = eventIterator.next(); int taskIndex = tezEvent.getSourceInfo().getTaskAttemptID().getTaskID().getId(); int taskAttemptIndex = tezEvent.getSourceInfo().getTaskAttemptID().getId(); if (taskIndex == taskId.getId()) { toForwardEvents.add((InputInitializerEvent) tezEvent.getEvent());
private void processOnDemandEvent(TezEvent tezEvent, Edge srcEdge, int srcTaskIndex) { onDemandRouteEventsWriteLock.lock(); try { if (tezEvent.getEventType() == EventType.DATA_MOVEMENT_EVENT || tezEvent.getEventType() == EventType.COMPOSITE_DATA_MOVEMENT_EVENT) { if (failedTaskAttemptIDs.contains(tezEvent.getSourceInfo().getTaskAttemptID())) { return; if (tezEvent.getEventType() == EventType.INPUT_FAILED_EVENT) { for (EventInfo eventInfo : onDemandRouteEvents) { if (eventInfo.eventEdge == srcEdge && eventInfo.tezEvent.getSourceInfo().getTaskAttemptID().equals( tezEvent.getSourceInfo().getTaskAttemptID()) && (eventInfo.tezEvent.getEventType() == EventType.DATA_MOVEMENT_EVENT || eventInfo.tezEvent .getEventType() == EventType.COMPOSITE_DATA_MOVEMENT_EVENT)) { failedTaskAttemptIDs.add(tezEvent.getSourceInfo().getTaskAttemptID());
@Override public void readFields(DataInput in) throws IOException { lastRequestId = in.readLong(); shouldDie = in.readBoolean(); nextFromEventId = in.readInt(); nextPreRoutedEventId = in.readInt(); if(in.readBoolean()) { int eventCount = in.readInt(); events = new ArrayList<TezEvent>(eventCount); for (int i = 0; i < eventCount; ++i) { TezEvent e = new TezEvent(); e.readFields(in); events.add(e); } } }
EventType eventType = tezEvent.getEventType(); switch (eventType) { case TASK_ATTEMPT_COMPLETED_EVENT:
@Override public void setLastEventSent(TezEvent lastEventSent) { writeLock.lock(); try { // TEZ-3066 ideally Heartbeat just happens in FAIL_IN_PROGRESS & KILL_IN_PROGRESS, // add other states here just in case. create TEZ-3068 for a more elegant solution. if (!EnumSet.of(TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptStateInternal.FAILED, TaskAttemptStateInternal.KILLED, TaskAttemptStateInternal.SUCCEEDED).contains(getInternalState())) { DataEventDependencyInfo info = new DataEventDependencyInfo( lastEventSent.getEventReceivedTime(), lastEventSent.getSourceInfo().getTaskAttemptID()); // task attempt id may be null for input data information events if (appendNextDataEvent) { appendNextDataEvent = false; lastDataEvents.add(info); } else { // over-write last event - array list makes it quick lastDataEvents.set(lastDataEvents.size() - 1, info); } } } finally { writeLock.unlock(); } } }
TaskAttemptEventOutputFailed destinationEvent = (TaskAttemptEventOutputFailed) castEvent.getCausalEvent(); schedulingCausalTA = destinationEvent.getInputFailedEvent().getSourceInfo().getTaskAttemptID();
@Override public void readFields(DataInput in) throws IOException { deserializeEvent(in); if (in.readBoolean()) { sourceInfo = new EventMetaData(); sourceInfo.readFields(in); } if (in.readBoolean()) { destinationInfo = new EventMetaData(); destinationInfo.readFields(in); } }
LOG.debug("Handling TezEvent in task" + ", taskAttemptId=" + taskSpec.getTaskAttemptID() + ", eventType=" + e.getEventType() + ", eventSourceInfo=" + e.getSourceInfo() + ", eventDestinationInfo=" + e.getDestinationInfo()); switch (e.getDestinationInfo().getEventGenerator()) { case INPUT: LogicalInput input = inputsMap.get( e.getDestinationInfo().getEdgeVertexName()); if (input != null) { ((InputFrameworkInterface)input).handleEvents(Collections.singletonList(e.getEvent())); } else { throw new TezUncheckedException("Unhandled event for invalid target: " case OUTPUT: LogicalOutput output = outputsMap.get( e.getDestinationInfo().getEdgeVertexName()); if (output != null) { ((OutputFrameworkInterface)output).handleEvents(Collections.singletonList(e.getEvent())); } else { throw new TezUncheckedException("Unhandled event for invalid target: " processor.handleEvents(Collections.singletonList(e.getEvent())); break; case SYSTEM: registerError(); EventMetaData sourceInfo = new EventMetaData(
Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time"); Event event = tezEvent.getEvent(); e = InputFailedEvent.create(inputIndex, ifEvent.getVersion()); tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime()); tezEventToSend.setDestinationInfo(destinationMetaInfo);
public void handleInputInitializerEvents(Collection<TezEvent> tezEvents) { List<InputInitializerEvent> toForwardEvents = new LinkedList<InputInitializerEvent>(); for (TezEvent tezEvent : tezEvents) { String srcVertexName = tezEvent.getSourceInfo().getTaskVertexName(); int taskIndex = tezEvent.getSourceInfo().getTaskAttemptID().getTaskID().getId(); int taskAttemptIndex = tezEvent.getSourceInfo().getTaskAttemptID().getId(); toForwardEvents.add((InputInitializerEvent) tezEvent.getEvent());
@Override public void readFields(DataInput in) throws IOException { if (in.readBoolean()) { int eventsCount = in.readInt(); events = new ArrayList<TezEvent>(eventsCount); for (int i = 0; i < eventsCount; ++i) { TezEvent e = new TezEvent(); e.readFields(in); events.add(e); } } if (in.readBoolean()) { currentTaskAttemptID = TezTaskAttemptID.readTezTaskAttemptID(in); } else { currentTaskAttemptID = null; } startIndex = in.readInt(); preRoutedStartIndex = in.readInt(); maxEvents = in.readInt(); requestId = in.readLong(); containerIdentifier = Text.readString(in); }