public HeartbeatCallable( SchedulerFragmentCompletingListener completionListener, RuntimeTask task, LlapTaskUmbilicalProtocol umbilical, long amPollInterval, long sendCounterInterval, int maxEventsToGet, AtomicLong requestCounter, String containerIdStr, TezEvent initialEvent, String fragmentRequestId, WmFragmentCounters wmCounters) { this.pollInterval = amPollInterval; this.sendCounterInterval = sendCounterInterval; this.maxEventsToGet = maxEventsToGet; this.requestCounter = requestCounter; this.containerIdStr = containerIdStr; this.initialEvent = initialEvent; this.completionListener = completionListener; this.fragmentRequestId = fragmentRequestId; this.wmCounters = wmCounters; this.task = task; this.umbilical = umbilical; this.updateEventMetadata = new EventMetaData(EventProducerConsumerType.SYSTEM, task.getVertexName(), "", task.getTaskAttemptID()); nextHeartbeatNumToLog = (Math.max(1, (int) (LOG_COUNTER_START_INTERVAL / (amPollInterval == 0 ? 0.000001f : (float) amPollInterval)))); }
switch (e.getDestinationInfo().getEventGenerator()) { case INPUT: LogicalInput input = inputsMap.get( e.getDestinationInfo().getEdgeVertexName()); if (input != null) { ((InputFrameworkInterface)input).handleEvents(Collections.singletonList(e.getEvent())); case OUTPUT: LogicalOutput output = outputsMap.get( e.getDestinationInfo().getEdgeVertexName()); if (output != null) { ((OutputFrameworkInterface)output).handleEvents(Collections.singletonList(e.getEvent())); LOG.warn("Failed to handle event", t); registerError(); EventMetaData sourceInfo = new EventMetaData( e.getDestinationInfo().getEventGenerator(), taskSpec.getVertexName(), e.getDestinationInfo().getEdgeVertexName(), getTaskAttemptID()); setFrameworkCounters();
public static RecoveryProtos.EventMetaDataProto convertEventMetaDataToProto( EventMetaData eventMetaData) { RecoveryProtos.EventMetaDataProto.Builder builder = RecoveryProtos.EventMetaDataProto.newBuilder() .setProducerConsumerType(eventMetaData.getEventGenerator().ordinal()) .setEdgeVertexName(eventMetaData.getEdgeVertexName()) .setTaskVertexName(eventMetaData.getTaskVertexName()); if (eventMetaData.getTaskAttemptID() != null) { builder.setTaskAttemptId(eventMetaData.getTaskAttemptID().toString()); } return builder.build(); }
@Override public void readFields(DataInput in) throws IOException { deserializeEvent(in); if (in.readBoolean()) { sourceInfo = new EventMetaData(); sourceInfo.readFields(in); } if (in.readBoolean()) { destinationInfo = new EventMetaData(); destinationInfo.readFields(in); } }
private void sendTaskGeneratedEvents(List<Event> events, EventProducerConsumerType generator, String taskVertexName, String edgeVertexName, TezTaskAttemptID taskAttemptID) { if (events == null || events.isEmpty()) { return; } EventMetaData eventMetaData = new EventMetaData(generator, taskVertexName, edgeVertexName, taskAttemptID); List<TezEvent> tezEvents = new ArrayList<TezEvent>(events.size()); for (Event e : events) { TezEvent te = new TezEvent(e, eventMetaData); tezEvents.add(te); } if (LOG.isDebugEnabled()) { for (TezEvent e : tezEvents) { LOG.debug("Generated event info" + ", eventMetaData=" + eventMetaData.toString() + ", eventType=" + e.getEventType()); } } tezUmbilical.addEvents(tezEvents); }
Task targetTask = getTask(destinationMeta.getTaskAttemptID().getTaskID()); targetTask.registerTezEvent(tezEvent); TezTaskAttemptID srcTaId = sourceMeta.getTaskAttemptID(); if (tezEvent.getEventType() == EventType.DATA_MOVEMENT_EVENT) { ((DataMovementEvent) tezEvent.getEvent()).setVersion(srcTaId.getId()); Vertex destVertex = getDAG().getVertex(sourceMeta.getEdgeVertexName()); Edge destEdge = targetVertices.get(destVertex); if (destEdge == null) { throw new TezUncheckedException("Bad destination vertex: " + sourceMeta.getEdgeVertexName() + " for event vertex: " + getLogIdentifier()); int srcTaskIndex = sourceMeta.getTaskAttemptID().getTaskID().getId(); Vertex edgeVertex = getDAG().getVertex(sourceMeta.getTaskVertexName()); Edge srcEdge = sourceVertices.get(edgeVertex); if (srcEdge == null) { throw new TezUncheckedException("Bad source vertex: " + sourceMeta.getTaskVertexName() + " for destination vertex: " + getLogIdentifier()); Preconditions.checkArgument(target != null, "Event sent to unkown vertex: " + vmEvent.getTargetVertexName()); TezTaskAttemptID srcTaId = sourceMeta.getTaskAttemptID(); if (srcTaId.getTaskID().getVertexID().equals(vertexId)) { Preconditions.checkArgument(target != null, "Event sent to unknown vertex: " + riEvent.getTargetVertexName());
case TASK_ATTEMPT_KILLED_EVENT: TaskAttemptTerminationCause errCause = null; switch (sourceMeta.getEventGenerator()) { case INPUT: errCause = TaskAttemptTerminationCause.INPUT_READ_ERROR; default: throw new TezUncheckedException("Unknown EventProducerConsumerType: " + sourceMeta.getEventGenerator()); new TaskAttemptEventAttemptFailed(sourceMeta.getTaskAttemptID(), TaskAttemptEventType.TA_FAILED, taskFailedEvent.getTaskFailureType(), "Error: " + taskFailedEvent.getDiagnostics(), TaskAttemptKilledEvent taskKilledEvent = (TaskAttemptKilledEvent) e.getEvent(); sendEvent( new TaskAttemptEventAttemptKilled(sourceMeta.getTaskAttemptID(), "Error: " + taskKilledEvent.getDiagnostics(), errCause)); case TASK_ATTEMPT_COMPLETED_EVENT: sendEvent( new TaskAttemptEvent(sourceMeta.getTaskAttemptID(), TaskAttemptEventType.TA_DONE)); break; default:
@Override public void setLastEventSent(TezEvent lastEventSent) { writeLock.lock(); try { // TEZ-3066 ideally Heartbeat just happens in FAIL_IN_PROGRESS & KILL_IN_PROGRESS, // add other states here just in case. create TEZ-3068 for a more elegant solution. if (!EnumSet.of(TaskAttemptStateInternal.FAIL_IN_PROGRESS, TaskAttemptStateInternal.KILL_IN_PROGRESS, TaskAttemptStateInternal.FAILED, TaskAttemptStateInternal.KILLED, TaskAttemptStateInternal.SUCCEEDED).contains(getInternalState())) { DataEventDependencyInfo info = new DataEventDependencyInfo( lastEventSent.getEventReceivedTime(), lastEventSent.getSourceInfo().getTaskAttemptID()); // task attempt id may be null for input data information events if (appendNextDataEvent) { appendNextDataEvent = false; lastDataEvents.add(info); } else { // over-write last event - array list makes it quick lastDataEvents.set(lastDataEvents.size() - 1, info); } } } finally { writeLock.unlock(); } } }
public void handleInputInitializerEvents(Collection<TezEvent> tezEvents) { List<InputInitializerEvent> toForwardEvents = new LinkedList<InputInitializerEvent>(); for (TezEvent tezEvent : tezEvents) { String srcVertexName = tezEvent.getSourceInfo().getTaskVertexName(); int taskIndex = tezEvent.getSourceInfo().getTaskAttemptID().getTaskID().getId(); int taskAttemptIndex = tezEvent.getSourceInfo().getTaskAttemptID().getId();
private static boolean isEventFromVertex(Vertex vertex, EventMetaData sourceMeta) { if (!sourceMeta.getTaskVertexName().equals(vertex.getName())) { return false; } return true; }
tezEvent.getEventType() == EventType.COMPOSITE_DATA_MOVEMENT_EVENT) { if (failedTaskAttemptIDs.contains(tezEvent.getSourceInfo().getTaskAttemptID())) { return; for (EventInfo eventInfo : onDemandRouteEvents) { if (eventInfo.eventEdge == srcEdge && eventInfo.tezEvent.getSourceInfo().getTaskAttemptID().equals( tezEvent.getSourceInfo().getTaskAttemptID()) && (eventInfo.tezEvent.getEventType() == EventType.DATA_MOVEMENT_EVENT || eventInfo.tezEvent failedTaskAttemptIDs.add(tezEvent.getSourceInfo().getTaskAttemptID());
public static TezEvent toTezEvent(NotTezEvent nte) throws InvalidProtocolBufferException { EventMetaData sourceMetaData = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, nte.getVertexName(), "NULL_VERTEX", null); EventMetaData destMetaData = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, nte.getVertexName(), nte.getDestInputName(), null); InputDataInformationEvent event = ProtoConverters.convertRootInputDataInformationEventFromProto( RootInputDataInformationEventProto.parseFrom(nte.getInputEventProtoBytes())); TezEvent tezEvent = new TezEvent(event, sourceMetaData, System.currentTimeMillis()); tezEvent.setDestinationInfo(destMetaData); return tezEvent; } }
while (eventIterator.hasNext()) { TezEvent tezEvent = eventIterator.next(); int taskIndex = tezEvent.getSourceInfo().getTaskAttemptID().getTaskID().getId(); int taskAttemptIndex = tezEvent.getSourceInfo().getTaskAttemptID().getId(); if (taskIndex == taskId.getId()) {
private EventMetaData getDestinationMetaData(String inputName) { EventMetaData destMeta = destinationEventMetadataMap.get(inputName); if (destMeta == null) { destMeta = new EventMetaData(EventProducerConsumerType.INPUT, managedVertex.getName(), inputName, null); destinationEventMetadataMap.put(inputName, destMeta); } return destMeta; }
(TaskAttemptEventOutputFailed) event; TezEvent tezEvent = outputFailedEvent.getInputFailedEvent(); TezTaskAttemptID failedDestTaId = tezEvent.getSourceInfo().getTaskAttemptID(); InputReadErrorEvent readErrorEvent = (InputReadErrorEvent)tezEvent.getEvent(); int failedInputIndexOnDestTa = readErrorEvent.getIndex();
public TezProcessorContextImpl(Configuration conf, String[] workDirs, int appAttemptNumber, TezUmbilical tezUmbilical, String dagName, String vertexName, int vertexParallelism, TezTaskAttemptID taskAttemptID, @Nullable UserPayload userPayload, LogicalIOProcessorRuntimeTask runtimeTask, Map<String, ByteBuffer> serviceConsumerMetadata, Map<String, String> auxServiceEnv, MemoryDistributor memDist, ProcessorDescriptor processorDescriptor, InputReadyTracker inputReadyTracker, ObjectRegistry objectRegistry, ExecutionContext ExecutionContext, long memAvailable, TezExecutors sharedExecutor) { super(conf, workDirs, appAttemptNumber, dagName, vertexName, vertexParallelism, taskAttemptID, runtimeTask.addAndGetTezCounter(vertexName), runtimeTask, tezUmbilical, serviceConsumerMetadata, auxServiceEnv, memDist, processorDescriptor, objectRegistry, ExecutionContext, memAvailable, sharedExecutor); checkNotNull(inputReadyTracker, "inputReadyTracker is null"); this.userPayload = userPayload; this.sourceInfo = new EventMetaData(EventProducerConsumerType.PROCESSOR, taskVertexName, "", taskAttemptID); this.inputReadyTracker = inputReadyTracker; }
InputReadErrorEvent event = (InputReadErrorEvent) tezEvent.getEvent(); TezTaskAttemptID destAttemptId = tezEvent.getSourceInfo() .getTaskAttemptID(); int destTaskIndex = destAttemptId.getTaskID().getId(); int srcTaskIndex;
public HeartbeatCallable(RuntimeTask task, TezTaskUmbilicalProtocol umbilical, long amPollInterval, long sendCounterInterval, int maxEventsToGet, AtomicLong requestCounter, String containerIdStr) { this.pollInterval = amPollInterval; this.sendCounterInterval = sendCounterInterval; this.maxEventsToGet = maxEventsToGet; this.requestCounter = requestCounter; this.containerIdStr = containerIdStr; this.task = task; this.umbilical = umbilical; this.updateEventMetadata = new EventMetaData(EventProducerConsumerType.SYSTEM, task.getVertexName(), "", task.getTaskAttemptID()); nextHeartbeatNumToLog = (Math.max(1, (int) (LOG_COUNTER_START_INTERVAL / (amPollInterval == 0 ? 0.000001f : (float) amPollInterval)))); }
.newHashMap(); TezTaskAttemptID srcAttemptId = tezEvent.getSourceInfo() .getTaskAttemptID(); int srcTaskIndex = srcAttemptId.getTaskID().getId();
public HeartbeatCallable( SchedulerFragmentCompletingListener completionListener, RuntimeTask task, LlapTaskUmbilicalProtocol umbilical, long amPollInterval, long sendCounterInterval, int maxEventsToGet, AtomicLong requestCounter, String containerIdStr, TezEvent initialEvent, String fragmentRequestId, WmFragmentCounters wmCounters) { this.pollInterval = amPollInterval; this.sendCounterInterval = sendCounterInterval; this.maxEventsToGet = maxEventsToGet; this.requestCounter = requestCounter; this.containerIdStr = containerIdStr; this.initialEvent = initialEvent; this.completionListener = completionListener; this.fragmentRequestId = fragmentRequestId; this.wmCounters = wmCounters; this.task = task; this.umbilical = umbilical; this.updateEventMetadata = new EventMetaData(EventProducerConsumerType.SYSTEM, task.getVertexName(), "", task.getTaskAttemptID()); nextHeartbeatNumToLog = (Math.max(1, (int) (LOG_COUNTER_START_INTERVAL / (amPollInterval == 0 ? 0.000001f : (float) amPollInterval)))); }