public int compare(final Pair<OperatorProfile, Integer> o1, final Pair<OperatorProfile, Integer> o2) { return Long.compare(o1.getLeft().getWaitNanos(), o2.getLeft().getWaitNanos()); } };
setWaitNanos(other.getWaitNanos());
setupSum += profile.getSetupNanos(); processSum += profile.getProcessNanos(); waitSum += profile.getWaitNanos(); memSum += profile.getPeakLocalMemoryAllocated(); tb.appendNanos(shortWait.getLeft().getWaitNanos()); tb.appendNanos(Math.round(waitSum / size)); tb.appendNanos(longWait.getLeft().getWaitNanos());
totalTimeInMillis += toMillis(operatorProfile.getProcessNanos() + operatorProfile.getWaitNanos() + operatorProfile.getSetupNanos()); final CoreOperatorType operatorType = CoreOperatorType.valueOf(operatorProfile.getOperatorType()); setWaitInClient(toMillis(operatorProfile.getWaitNanos())); toMillis(operatorProfile.getProcessNanos() + operatorProfile.getWaitNanos() + operatorProfile.getSetupNanos())); break; toMillis(operatorProfile.getSetupNanos() + operatorProfile.getProcessNanos() + operatorProfile.getWaitNanos())); break;
totalNanos += op.getSetupNanos(); totalNanos += op.getProcessNanos(); totalNanos += op.getWaitNanos();
private void setCommonDatasetProfile(CommonDatasetProfile datasetProfile, OperatorProfile operatorProfile, MajorFragmentProfile majorFragment, long inputBytes, long inputRecords, List<String> datasetPath) { if (datasetProfile.getBytesRead() != null) { datasetProfile.setBytesRead(datasetProfile.getRecordsRead() + inputBytes); } else { datasetProfile.setBytesRead(inputBytes); } if (datasetProfile.getRecordsRead() != null) { datasetProfile.setRecordsRead(datasetProfile.getRecordsRead() + inputRecords); } else { datasetProfile.setRecordsRead(inputRecords); } final DatasetPathUI datasetPathUI = new DatasetPathUI().setDatasetPathList(datasetPath); if (!datasetProfile.getDatasetPathsList().contains(datasetPathUI)) { datasetProfile.getDatasetPathsList().add(datasetPathUI); } if (datasetProfile.getWaitOnSource() != null) { datasetProfile.setWaitOnSource(datasetProfile.getWaitOnSource() + toMillis(operatorProfile.getWaitNanos())); } else { datasetProfile.setWaitOnSource(toMillis(operatorProfile.getWaitNanos())); } if (datasetProfile.getParallelism() != null) { datasetProfile.setParallelism(datasetProfile.getParallelism() + 1); } else { datasetProfile.setParallelism(1); } }
@Test public void testReadIOStats() throws Exception { FileSystemWrapper dfs = null; InputStream is = null; Configuration conf = new Configuration(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); OpProfileDef profileDef = new OpProfileDef(0 /*operatorId*/, 0 /*operatorType*/, 0 /*inputCount*/); OperatorStats stats = new OperatorStats(profileDef, null /*allocator*/); // start wait time method in OperatorStats expects the OperatorStats state to be in "processing" stats.startProcessing(); try { dfs = new FileSystemWrapper(conf, stats, null); is = dfs.open(new Path(tempFilePath)); byte[] buf = new byte[8000]; while (is.read(buf, 0, buf.length) != -1) { } } finally { stats.stopProcessing(); if (is != null) { is.close(); } if (dfs != null) { dfs.close(); } } OperatorProfile operatorProfile = stats.getProfile(); assertTrue("Expected wait time is non-zero, but got zero wait time", operatorProfile.getWaitNanos() > 0); }
assertTrue("Expected wait time is non-zero, but got zero wait time", operatorProfile.getWaitNanos() > 0);
public void writeTo(io.protostuff.Output output, com.dremio.exec.proto.UserBitShared.OperatorProfile message) throws java.io.IOException { for(com.dremio.exec.proto.UserBitShared.StreamProfile inputProfile : message.getInputProfileList()) output.writeObject(1, inputProfile, com.dremio.exec.proto.SchemaUserBitShared.StreamProfile.WRITE, true); if(message.hasOperatorId()) output.writeInt32(3, message.getOperatorId(), false); if(message.hasOperatorType()) output.writeInt32(4, message.getOperatorType(), false); if(message.hasSetupNanos()) output.writeInt64(5, message.getSetupNanos(), false); if(message.hasProcessNanos()) output.writeInt64(6, message.getProcessNanos(), false); if(message.hasPeakLocalMemoryAllocated()) output.writeInt64(7, message.getPeakLocalMemoryAllocated(), false); for(com.dremio.exec.proto.UserBitShared.MetricValue metric : message.getMetricList()) output.writeObject(8, metric, com.dremio.exec.proto.SchemaUserBitShared.MetricValue.WRITE, true); if(message.hasWaitNanos()) output.writeInt64(9, message.getWaitNanos(), false); } public boolean isInitialized(com.dremio.exec.proto.UserBitShared.OperatorProfile message)
public String getContent() { TableBuilder builder = new TableBuilder(OPERATOR_COLUMNS); for (ImmutablePair<OperatorProfile, Integer> ip : ops) { int minor = ip.getRight(); OperatorProfile op = ip.getLeft(); String path = new OperatorPathBuilder().setMajor(major).setMinor(minor).setOperator(op).build(); builder.appendCell(path, null); builder.appendNanos(op.getSetupNanos()); builder.appendNanos(op.getProcessNanos()); builder.appendNanos(op.getWaitNanos()); long maxBatches = Long.MIN_VALUE; long maxRecords = Long.MIN_VALUE; for (StreamProfile sp : op.getInputProfileList()) { maxBatches = Math.max(sp.getBatches(), maxBatches); maxRecords = Math.max(sp.getRecords(), maxRecords); } builder.appendFormattedInteger(maxBatches, null); builder.appendFormattedInteger(maxRecords, null); builder.appendBytes(op.getPeakLocalMemoryAllocated(), null); } return builder.build(); }