/** * Sends out final events for task success. * @param taskAttemptID * @return * @throws IOException * indicates an RPC communication failure. * @throws TezException * indicates an exception somewhere in the AM. */ private boolean taskSucceeded(TezTaskAttemptID taskAttemptID) throws IOException, TezException { // Ensure only one final event is ever sent. if (!finalEventQueued.getAndSet(true)) { TezEvent statusUpdateEvent = new TezEvent( getStatusUpdateEvent(true, true), updateEventMetadata); TezEvent taskCompletedEvent = new TezEvent(new TaskAttemptCompletedEvent(), updateEventMetadata); if (LOG.isDebugEnabled()) { LOG.debug("Invoking OOB heartbeat for successful attempt: {}, isTaskDone={}", taskAttemptID, task.isTaskDone()); } completionListener.fragmentCompleting(fragmentRequestId, SchedulerFragmentCompletingListener.State.SUCCESS); return !heartbeat(Lists.newArrayList(statusUpdateEvent, taskCompletedEvent)).shouldDie; } else { LOG.warn("A final task state event has already been sent. Not sending again"); return askedToDie.get(); } }
tezEvents.add(new TezEvent(new TaskAttemptKilledEvent(diagnostics), srcMeta == null ? updateEventMetadata : srcMeta)); } else { tezEvents.add(new TezEvent(new TaskAttemptFailedEvent(diagnostics, taskFailureType), srcMeta == null ? updateEventMetadata : srcMeta)); tezEvents.add(new TezEvent( getStatusUpdateEvent(true, true), updateEventMetadata)); } catch (Exception e) {
public static TezEvent toTezEvent(NotTezEvent nte) throws InvalidProtocolBufferException { EventMetaData sourceMetaData = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, nte.getVertexName(), "NULL_VERTEX", null); EventMetaData destMetaData = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, nte.getVertexName(), nte.getDestInputName(), null); InputDataInformationEvent event = ProtoConverters.convertRootInputDataInformationEventFromProto( RootInputDataInformationEventProto.parseFrom(nte.getInputEventProtoBytes())); TezEvent tezEvent = new TezEvent(event, sourceMetaData, System.currentTimeMillis()); tezEvent.setDestinationInfo(destMetaData); return tezEvent; } }
prevCounterSendHeartbeatNum = nonOobHeartbeatCounter.get(); updateEvent = new TezEvent(getStatusUpdateEvent(sendCounters, false), updateEventMetadata); events.add(updateEvent);
@Override public void sendEvents(List<Event> events) { Preconditions.checkNotNull(events, "events are null"); List<TezEvent> tezEvents = new ArrayList<TezEvent>(events.size()); for (Event e : events) { TezEvent tEvt = new TezEvent(e, sourceInfo); tezEvents.add(tEvt); } tezUmbilical.addEvents(tezEvents); }
@Override public void sendEvents(List<Event> events) { Preconditions.checkNotNull(events, "events are null"); List<TezEvent> tezEvents = new ArrayList<TezEvent>(events.size()); for (Event e : events) { TezEvent tEvt = new TezEvent(e, sourceInfo); tezEvents.add(tEvt); } tezUmbilical.addEvents(tezEvents); }
@Override public void sendEvents(List<Event> events) { Preconditions.checkNotNull(events, "events are null"); List<TezEvent> tezEvents = new ArrayList<TezEvent>(events.size()); for (Event e : events) { TezEvent tEvt = new TezEvent(e, sourceInfo); tezEvents.add(tEvt); } tezUmbilical.addEvents(tezEvents); }
@Override public TezEvent apply(InputDataInformationEvent riEvent) { TezEvent tezEvent = new TezEvent(riEvent, rootEventSourceMetadata, currTime); tezEvent.setDestinationInfo(getDestinationMetaData(inputName)); return tezEvent; } });
@Override public void readFields(DataInput in) throws IOException { lastRequestId = in.readLong(); shouldDie = in.readBoolean(); nextFromEventId = in.readInt(); nextPreRoutedEventId = in.readInt(); if(in.readBoolean()) { int eventCount = in.readInt(); events = new ArrayList<TezEvent>(eventCount); for (int i = 0; i < eventCount; ++i) { TezEvent e = new TezEvent(); e.readFields(in); events.add(e); } } }
/** * Sends out final events for task success. * @param taskAttemptID * @return * @throws IOException * indicates an RPC communication failure. * @throws TezException * indicates an exception somewhere in the AM. */ private boolean taskSucceeded(TezTaskAttemptID taskAttemptID) throws IOException, TezException { // Ensure only one final event is ever sent. if (!finalEventQueued.getAndSet(true)) { TezEvent statusUpdateEvent = new TezEvent(getStatusUpdateEvent(true), updateEventMetadata); TezEvent taskCompletedEvent = new TezEvent(new TaskAttemptCompletedEvent(), updateEventMetadata); return !heartbeat(Lists.newArrayList(statusUpdateEvent, taskCompletedEvent)).shouldDie; } else { LOG.warn("A final task state event has already been sent. Not sending again"); return askedToDie.get(); } }
@Override public void readFields(DataInput in) throws IOException { if (in.readBoolean()) { int eventsCount = in.readInt(); events = new ArrayList<TezEvent>(eventsCount); for (int i = 0; i < eventsCount; ++i) { TezEvent e = new TezEvent(); e.readFields(in); events.add(e); } } if (in.readBoolean()) { currentTaskAttemptID = TezTaskAttemptID.readTezTaskAttemptID(in); } else { currentTaskAttemptID = null; } startIndex = in.readInt(); preRoutedStartIndex = in.readInt(); maxEvents = in.readInt(); requestId = in.readLong(); containerIdentifier = Text.readString(in); }
private void sendTaskGeneratedEvents(List<Event> events, EventProducerConsumerType generator, String taskVertexName, String edgeVertexName, TezTaskAttemptID taskAttemptID) { if (events == null || events.isEmpty()) { return; } EventMetaData eventMetaData = new EventMetaData(generator, taskVertexName, edgeVertexName, taskAttemptID); List<TezEvent> tezEvents = new ArrayList<TezEvent>(events.size()); for (Event e : events) { TezEvent te = new TezEvent(e, eventMetaData); tezEvents.add(te); } if (LOG.isDebugEnabled()) { for (TezEvent e : tezEvents) { LOG.debug("Generated event info" + ", eventMetaData=" + eventMetaData.toString() + ", eventType=" + e.getEventType()); } } tezUmbilical.addEvents(tezEvents); }
/** * Sends out final events for task success. * @param taskAttemptID * @return * @throws IOException * indicates an RPC communication failure. * @throws TezException * indicates an exception somewhere in the AM. */ private boolean taskSucceeded(TezTaskAttemptID taskAttemptID) throws IOException, TezException { // Ensure only one final event is ever sent. if (!finalEventQueued.getAndSet(true)) { TezEvent statusUpdateEvent = new TezEvent( getStatusUpdateEvent(true, true), updateEventMetadata); TezEvent taskCompletedEvent = new TezEvent(new TaskAttemptCompletedEvent(), updateEventMetadata); if (LOG.isDebugEnabled()) { LOG.debug("Invoking OOB heartbeat for successful attempt: {}, isTaskDone={}", taskAttemptID, task.isTaskDone()); } completionListener.fragmentCompleting(fragmentRequestId, SchedulerFragmentCompletingListener.State.SUCCESS); return !heartbeat(Lists.newArrayList(statusUpdateEvent, taskCompletedEvent)).shouldDie; } else { LOG.warn("A final task state event has already been sent. Not sending again"); return askedToDie.get(); } }
@Override public synchronized void addRootInputEvents(final String inputName, Collection<InputDataInformationEvent> events) { checkAndThrowIfDone(); verifyIsRootInput(inputName); final long currTime = appContext.getClock().getTime(); Collection<TezEvent> tezEvents = Collections2.transform(events, new Function<InputDataInformationEvent, TezEvent>() { @Override public TezEvent apply(InputDataInformationEvent riEvent) { TezEvent tezEvent = new TezEvent(riEvent, rootEventSourceMetadata, currTime); tezEvent.setDestinationInfo(getDestinationMetaData(inputName)); return tezEvent; } }); if (LOG.isDebugEnabled()) { LOG.debug("vertex:" + managedVertex.getLogIdentifier() + "; Added " + events.size() + " for input " + "name " + inputName); } rootInputInitEventQueue.addAll(tezEvents); // Recovery handling is taken care of by the Vertex. }
@VisibleForTesting protected void sendInputFailedToConsumers() { Vertex vertex = getVertex(); Map<Vertex, Edge> edges = vertex.getOutputVertices(); if (edges != null && !edges.isEmpty()) { List<TezEvent> tezIfEvents = Lists.newArrayListWithCapacity(edges.size()); for (Vertex edgeVertex : edges.keySet()) { tezIfEvents.add(new TezEvent(new InputFailedEvent(), new EventMetaData(EventProducerConsumerType.SYSTEM, vertex.getName(), edgeVertex.getName(), getID()), appContext.getClock().getTime())); } sendEvent(new VertexEventRouteEvent(vertex.getVertexId(), tezIfEvents)); } }
private void handleCompositeDataMovementEvent(TezEvent tezEvent) throws AMUserCodeException { CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent(); EventMetaData srcInfo = tezEvent.getSourceInfo(); for (DataMovementEvent dmEvent : compEvent.getEvents()) { TezEvent newEvent = new TezEvent(dmEvent, srcInfo, tezEvent.getEventReceivedTime()); sendTezEventToDestinationTasks(newEvent); } }
e = InputFailedEvent.create(inputIndex, ifEvent.getVersion()); tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime()); tezEventToSend.setDestinationInfo(destinationMetaInfo);
public List<TezEvent> generateEmptyEventsForAttempt(TezTaskAttemptID attempt) throws Exception { if (!edgeProperty.getEdgeSource().getClassName().startsWith("org.apache.tez")) { throw new TezException("Only org.apache.tez outputs are allowed for max percent failure feature. Disallowed Output: " + edgeProperty.getEdgeSource().getClassName()); } List<Event> events = new ArrayList<>(); Deflater deflater = TezCommonUtils.newBestCompressionDeflater(); try { ShuffleUtils.generateEventsForNonStartedOutput(events, edgeManager.getNumDestinationConsumerTasks(attempt.getTaskID().getId()), null, false, true, deflater); } catch (Exception e) { throw new TezException(e); } EventMetaData sourceInfo = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, sourceVertex.getName(), getDestinationVertexName(), attempt); List<TezEvent> tezEvents = new ArrayList<>(events.size()); for (Event e : events) { TezEvent tezEvent = new TezEvent(e, sourceInfo); tezEvents.add(tezEvent); } return tezEvents; }
@Override public void sendEventToProcessor(Collection<CustomProcessorEvent> events, int taskId) { checkAndThrowIfDone(); Preconditions.checkArgument(taskId >= 0 && taskId < managedVertex.getTotalTasks(), "Invalid taskId " + taskId + "; " + "There are " + managedVertex.getTotalTasks() + " tasks in total."); if (events != null && events.size() > 0) { List<TezEvent> tezEvents = new ArrayList<>(); for (CustomProcessorEvent event : events) { TezEvent tezEvent = new TezEvent(event, null); // use dummy task attempt id since this is not an task attempt specific event and task // attempt id won't be used anyway EventMetaData destinationMeta = new EventMetaData(EventProducerConsumerType.PROCESSOR, managedVertex.getName(), managedVertex.getName(), TezTaskAttemptID.getInstance(managedVertex.getTask(taskId).getTaskId(), -1)); tezEvent.setDestinationInfo(destinationMeta); tezEvents.add(tezEvent); } appContext.getEventHandler().handle( new VertexEventRouteEvent(managedVertex.getVertexId(), tezEvents)); } }
destinationInfo = convertEventMetaDataFromProto(eventProto.getDestinationInfo()); TezEvent tezEvent = new TezEvent(evt, sourceInfo, eventProto.getEventTime()); tezEvent.setDestinationInfo(destinationInfo); return tezEvent;