@Override public synchronized void reportDataSize(long size) { // this is a concurrent map. Plus we are not adding/deleting entries runtimeTask.getTaskStatistics().getIOStatistics().get(sourceVertexName) .setDataSize(size); }
@Override public void readFields(DataInput in) throws IOException { progress = in.readFloat(); progressNotified = in.readBoolean(); if (in.readBoolean()) { tezCounters = new TezCounters(); tezCounters.readFields(in); } if (in.readBoolean()) { statistics = new TaskStatistics(); statistics.readFields(in); } }
public void addIO(String edgeName) { addIO(edgeName, new IOStatistics()); }
@Override public void write(DataOutput out) throws IOException { out.writeFloat(progress); out.writeBoolean(progressNotified); if (tezCounters != null) { out.writeBoolean(true); tezCounters.write(out); } else { out.writeBoolean(false); } if (statistics != null) { out.writeBoolean(true); statistics.write(out); } else { out.writeBoolean(false); } }
protected RuntimeTask(TaskSpec taskSpec, Configuration tezConf, TezUmbilical tezUmbilical, String pid, boolean setupSysCounterUpdater) { this.taskSpec = taskSpec; this.tezConf = tezConf; this.tezUmbilical = tezUmbilical; this.tezCounters = new TezCounters(); this.eventCounter = new AtomicInteger(0); this.nextFromEventId = new AtomicInteger(0); this.nextPreRoutedEventId = new AtomicInteger(0); this.progress = 0.0f; this.taskDone = new AtomicBoolean(false); this.statistics = new TaskStatistics(); if (setupSysCounterUpdater) { this.counterUpdater = new TaskCounterUpdater(tezCounters, tezConf, pid); } else { this.counterUpdater = null; } }
void mergeFrom(TaskStatistics taskStats) { if (taskStats == null) { return; } for (Map.Entry<String, org.apache.tez.runtime.api.impl.IOStatistics> entry : taskStats .getIOStatistics().entrySet()) { String ioName = entry.getKey(); IOStatisticsImpl myIOStat = ioStats.get(ioName); Preconditions.checkState(myIOStat != null, "Unexpected IO name: " + ioName + " for vertex:" + getLogIdentifier()); myIOStat.mergeFrom(entry.getValue()); } }
@Override public void readFields(DataInput in) throws IOException { int numEntries = in.readInt(); for (int i=0; i<numEntries; ++i) { String edgeName = Text.readString(in); IOStatistics edgeStats = new IOStatistics(); edgeStats.readFields(in); addIO(edgeName, edgeStats); } } }
@Override public void reportItemsProcessed(long items) { // this is a concurrent map. Plus we are not adding/deleting entries runtimeTask.getTaskStatistics().getIOStatistics().get(destinationVertexName) .setItemsProcessed(items);; }
this.inputs = inputs; this.inputReadyTracker = inputReadyTracker; runtimeTask.getTaskStatistics().addIO(sourceVertexName); statsReporter = new InputStatisticsReporterImpl();
@Override public synchronized void reportDataSize(long size) { // this is a concurrent map. Plus we are not adding/deleting entries runtimeTask.getTaskStatistics().getIOStatistics().get(destinationVertexName) .setDataSize(size); }
@Private public TezOutputContextImpl(Configuration conf, String[] workDirs, int appAttemptNumber, TezUmbilical tezUmbilical, String dagName, String taskVertexName, String destinationVertexName, int vertexParallelism, TezTaskAttemptID taskAttemptID, int outputIndex, @Nullable UserPayload userPayload, LogicalIOProcessorRuntimeTask runtimeTask, Map<String, ByteBuffer> serviceConsumerMetadata, Map<String, String> auxServiceEnv, MemoryDistributor memDist, OutputDescriptor outputDescriptor, ObjectRegistry objectRegistry, ExecutionContext executionContext, long memAvailable, TezExecutors sharedExecutor) { super(conf, workDirs, appAttemptNumber, dagName, taskVertexName, vertexParallelism, taskAttemptID, wrapCounters(runtimeTask, taskVertexName, destinationVertexName, conf), runtimeTask, tezUmbilical, serviceConsumerMetadata, auxServiceEnv, memDist, outputDescriptor, objectRegistry, executionContext, memAvailable, sharedExecutor); checkNotNull(outputIndex, "outputIndex is null"); checkNotNull(destinationVertexName, "destinationVertexName is null"); this.userPayload = userPayload; this.outputIndex = outputIndex; this.destinationVertexName = destinationVertexName; this.sourceInfo = new EventMetaData(EventProducerConsumerType.OUTPUT, taskVertexName, destinationVertexName, taskAttemptID); runtimeTask.getTaskStatistics().addIO(destinationVertexName); statsReporter = new OutputStatisticsReporterImpl(); }
@Override public void reportItemsProcessed(long items) { // this is a concurrent map. Plus we are not adding/deleting entries runtimeTask.getTaskStatistics().getIOStatistics().get(sourceVertexName) .setItemsProcessed(items); }