public HeartbeatCallable( SchedulerFragmentCompletingListener completionListener, RuntimeTask task, LlapTaskUmbilicalProtocol umbilical, long amPollInterval, long sendCounterInterval, int maxEventsToGet, AtomicLong requestCounter, String containerIdStr, TezEvent initialEvent, String fragmentRequestId, WmFragmentCounters wmCounters) { this.pollInterval = amPollInterval; this.sendCounterInterval = sendCounterInterval; this.maxEventsToGet = maxEventsToGet; this.requestCounter = requestCounter; this.containerIdStr = containerIdStr; this.initialEvent = initialEvent; this.completionListener = completionListener; this.fragmentRequestId = fragmentRequestId; this.wmCounters = wmCounters; this.task = task; this.umbilical = umbilical; this.updateEventMetadata = new EventMetaData(EventProducerConsumerType.SYSTEM, task.getVertexName(), "", task.getTaskAttemptID()); nextHeartbeatNumToLog = (Math.max(1, (int) (LOG_COUNTER_START_INTERVAL / (amPollInterval == 0 ? 0.000001f : (float) amPollInterval)))); }
public static TezEvent toTezEvent(NotTezEvent nte) throws InvalidProtocolBufferException { EventMetaData sourceMetaData = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, nte.getVertexName(), "NULL_VERTEX", null); EventMetaData destMetaData = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, nte.getVertexName(), nte.getDestInputName(), null); InputDataInformationEvent event = ProtoConverters.convertRootInputDataInformationEventFromProto( RootInputDataInformationEventProto.parseFrom(nte.getInputEventProtoBytes())); TezEvent tezEvent = new TezEvent(event, sourceMetaData, System.currentTimeMillis()); tezEvent.setDestinationInfo(destMetaData); return tezEvent; } }
private EventMetaData getDestinationMetaData(String inputName) { EventMetaData destMeta = destinationEventMetadataMap.get(inputName); if (destMeta == null) { destMeta = new EventMetaData(EventProducerConsumerType.INPUT, managedVertex.getName(), inputName, null); destinationEventMetadataMap.put(inputName, destMeta); } return destMeta; }
@Override public void readFields(DataInput in) throws IOException { deserializeEvent(in); if (in.readBoolean()) { sourceInfo = new EventMetaData(); sourceInfo.readFields(in); } if (in.readBoolean()) { destinationInfo = new EventMetaData(); destinationInfo.readFields(in); } }
public TezProcessorContextImpl(Configuration conf, String[] workDirs, int appAttemptNumber, TezUmbilical tezUmbilical, String dagName, String vertexName, int vertexParallelism, TezTaskAttemptID taskAttemptID, @Nullable UserPayload userPayload, LogicalIOProcessorRuntimeTask runtimeTask, Map<String, ByteBuffer> serviceConsumerMetadata, Map<String, String> auxServiceEnv, MemoryDistributor memDist, ProcessorDescriptor processorDescriptor, InputReadyTracker inputReadyTracker, ObjectRegistry objectRegistry, ExecutionContext ExecutionContext, long memAvailable, TezExecutors sharedExecutor) { super(conf, workDirs, appAttemptNumber, dagName, vertexName, vertexParallelism, taskAttemptID, runtimeTask.addAndGetTezCounter(vertexName), runtimeTask, tezUmbilical, serviceConsumerMetadata, auxServiceEnv, memDist, processorDescriptor, objectRegistry, ExecutionContext, memAvailable, sharedExecutor); checkNotNull(inputReadyTracker, "inputReadyTracker is null"); this.userPayload = userPayload; this.sourceInfo = new EventMetaData(EventProducerConsumerType.PROCESSOR, taskVertexName, "", taskAttemptID); this.inputReadyTracker = inputReadyTracker; }
public HeartbeatCallable(RuntimeTask task, TezTaskUmbilicalProtocol umbilical, long amPollInterval, long sendCounterInterval, int maxEventsToGet, AtomicLong requestCounter, String containerIdStr) { this.pollInterval = amPollInterval; this.sendCounterInterval = sendCounterInterval; this.maxEventsToGet = maxEventsToGet; this.requestCounter = requestCounter; this.containerIdStr = containerIdStr; this.task = task; this.umbilical = umbilical; this.updateEventMetadata = new EventMetaData(EventProducerConsumerType.SYSTEM, task.getVertexName(), "", task.getTaskAttemptID()); nextHeartbeatNumToLog = (Math.max(1, (int) (LOG_COUNTER_START_INTERVAL / (amPollInterval == 0 ? 0.000001f : (float) amPollInterval)))); }
public HeartbeatCallable( SchedulerFragmentCompletingListener completionListener, RuntimeTask task, LlapTaskUmbilicalProtocol umbilical, long amPollInterval, long sendCounterInterval, int maxEventsToGet, AtomicLong requestCounter, String containerIdStr, TezEvent initialEvent, String fragmentRequestId, WmFragmentCounters wmCounters) { this.pollInterval = amPollInterval; this.sendCounterInterval = sendCounterInterval; this.maxEventsToGet = maxEventsToGet; this.requestCounter = requestCounter; this.containerIdStr = containerIdStr; this.initialEvent = initialEvent; this.completionListener = completionListener; this.fragmentRequestId = fragmentRequestId; this.wmCounters = wmCounters; this.task = task; this.umbilical = umbilical; this.updateEventMetadata = new EventMetaData(EventProducerConsumerType.SYSTEM, task.getVertexName(), "", task.getTaskAttemptID()); nextHeartbeatNumToLog = (Math.max(1, (int) (LOG_COUNTER_START_INTERVAL / (amPollInterval == 0 ? 0.000001f : (float) amPollInterval)))); }
private void sendTaskGeneratedEvents(List<Event> events, EventProducerConsumerType generator, String taskVertexName, String edgeVertexName, TezTaskAttemptID taskAttemptID) { if (events == null || events.isEmpty()) { return; } EventMetaData eventMetaData = new EventMetaData(generator, taskVertexName, edgeVertexName, taskAttemptID); List<TezEvent> tezEvents = new ArrayList<TezEvent>(events.size()); for (Event e : events) { TezEvent te = new TezEvent(e, eventMetaData); tezEvents.add(te); } if (LOG.isDebugEnabled()) { for (TezEvent e : tezEvents) { LOG.debug("Generated event info" + ", eventMetaData=" + eventMetaData.toString() + ", eventType=" + e.getEventType()); } } tezUmbilical.addEvents(tezEvents); }
@VisibleForTesting protected void sendInputFailedToConsumers() { Vertex vertex = getVertex(); Map<Vertex, Edge> edges = vertex.getOutputVertices(); if (edges != null && !edges.isEmpty()) { List<TezEvent> tezIfEvents = Lists.newArrayListWithCapacity(edges.size()); for (Vertex edgeVertex : edges.keySet()) { tezIfEvents.add(new TezEvent(new InputFailedEvent(), new EventMetaData(EventProducerConsumerType.SYSTEM, vertex.getName(), edgeVertex.getName(), getID()), appContext.getClock().getTime())); } sendEvent(new VertexEventRouteEvent(vertex.getVertexId(), tezIfEvents)); } }
this.inputIndex = inputIndex; this.sourceVertexName = sourceVertexName; this.sourceInfo = new EventMetaData( EventProducerConsumerType.INPUT, taskVertexName, sourceVertexName, taskAttemptID);
@Private public TezOutputContextImpl(Configuration conf, String[] workDirs, int appAttemptNumber, TezUmbilical tezUmbilical, String dagName, String taskVertexName, String destinationVertexName, int vertexParallelism, TezTaskAttemptID taskAttemptID, int outputIndex, @Nullable UserPayload userPayload, LogicalIOProcessorRuntimeTask runtimeTask, Map<String, ByteBuffer> serviceConsumerMetadata, Map<String, String> auxServiceEnv, MemoryDistributor memDist, OutputDescriptor outputDescriptor, ObjectRegistry objectRegistry, ExecutionContext executionContext, long memAvailable, TezExecutors sharedExecutor) { super(conf, workDirs, appAttemptNumber, dagName, taskVertexName, vertexParallelism, taskAttemptID, wrapCounters(runtimeTask, taskVertexName, destinationVertexName, conf), runtimeTask, tezUmbilical, serviceConsumerMetadata, auxServiceEnv, memDist, outputDescriptor, objectRegistry, executionContext, memAvailable, sharedExecutor); checkNotNull(outputIndex, "outputIndex is null"); checkNotNull(destinationVertexName, "destinationVertexName is null"); this.userPayload = userPayload; this.outputIndex = outputIndex; this.destinationVertexName = destinationVertexName; this.sourceInfo = new EventMetaData(EventProducerConsumerType.OUTPUT, taskVertexName, destinationVertexName, taskAttemptID); runtimeTask.getTaskStatistics().addIO(destinationVertexName); statsReporter = new OutputStatisticsReporterImpl(); }
public void initialize() throws AMUserCodeException { if (edgeManager != null) { try { edgeManager.initialize(); } catch (Exception e) { throw new AMUserCodeException(Source.EdgeManager, "Fail to initialize Edge," + getEdgeInfo(), e); } } synchronized (this) { destinationMetaInfo = new EventMetaData(EventProducerConsumerType.INPUT, destinationVertex.getName(), sourceVertex.getName(), null); } }
public List<TezEvent> generateEmptyEventsForAttempt(TezTaskAttemptID attempt) throws Exception { if (!edgeProperty.getEdgeSource().getClassName().startsWith("org.apache.tez")) { throw new TezException("Only org.apache.tez outputs are allowed for max percent failure feature. Disallowed Output: " + edgeProperty.getEdgeSource().getClassName()); } List<Event> events = new ArrayList<>(); Deflater deflater = TezCommonUtils.newBestCompressionDeflater(); try { ShuffleUtils.generateEventsForNonStartedOutput(events, edgeManager.getNumDestinationConsumerTasks(attempt.getTaskID().getId()), null, false, true, deflater); } catch (Exception e) { throw new TezException(e); } EventMetaData sourceInfo = new EventMetaData(EventMetaData.EventProducerConsumerType.INPUT, sourceVertex.getName(), getDestinationVertexName(), attempt); List<TezEvent> tezEvents = new ArrayList<>(events.size()); for (Event e : events) { TezEvent tezEvent = new TezEvent(e, sourceInfo); tezEvents.add(tezEvent); } return tezEvents; }
@Override public void sendEventToProcessor(Collection<CustomProcessorEvent> events, int taskId) { checkAndThrowIfDone(); Preconditions.checkArgument(taskId >= 0 && taskId < managedVertex.getTotalTasks(), "Invalid taskId " + taskId + "; " + "There are " + managedVertex.getTotalTasks() + " tasks in total."); if (events != null && events.size() > 0) { List<TezEvent> tezEvents = new ArrayList<>(); for (CustomProcessorEvent event : events) { TezEvent tezEvent = new TezEvent(event, null); // use dummy task attempt id since this is not an task attempt specific event and task // attempt id won't be used anyway EventMetaData destinationMeta = new EventMetaData(EventProducerConsumerType.PROCESSOR, managedVertex.getName(), managedVertex.getName(), TezTaskAttemptID.getInstance(managedVertex.getTask(taskId).getTaskId(), -1)); tezEvent.setDestinationInfo(destinationMeta); tezEvents.add(tezEvent); } appContext.getEventHandler().handle( new VertexEventRouteEvent(managedVertex.getVertexId(), tezEvents)); } }
public static EventMetaData convertEventMetaDataFromProto( RecoveryProtos.EventMetaDataProto proto) { TezTaskAttemptID attemptID = null; if (proto.hasTaskAttemptId()) { attemptID = TezTaskAttemptID.fromString(proto.getTaskAttemptId()); } return new EventMetaData( EventMetaData.EventProducerConsumerType.values()[proto.getProducerConsumerType()], proto.getTaskVertexName(), proto.getEdgeVertexName(), attemptID); } }
LOG.warn("Failed to handle event", t); registerError(); EventMetaData sourceInfo = new EventMetaData( e.getDestinationInfo().getEventGenerator(), taskSpec.getVertexName(), e.getDestinationInfo().getEdgeVertexName(),