private void setScanStats(CoreOperatorType operatorType, OperatorProfile operatorProfile, MajorFragmentProfile majorFragment) { final String operatorId = createOperatorKey(majorFragment.getMajorFragmentId(), operatorProfile.getOperatorId()); long inputBytes = 0L; long inputRecords = 0L; if (operatorProfile.getInputProfileList() != null) { for (StreamProfile stream : operatorProfile.getInputProfileList()) { inputRecords += stream.getRecords(); inputBytes += stream.getSize(); } addInputBytesAndRecords(inputBytes, inputRecords); } if (operatorToTable.containsKey(operatorId)) { // TODO check if its fs based or table based using operatorType final String tableName = operatorToTable.get(operatorId); TableDatasetProfile tableDatasetProfile = tableDatasetProfileMap.get(tableName); if (tableDatasetProfile == null) { tableDatasetProfile = new TableDatasetProfile(); tableDatasetProfile.setDatasetProfile(new CommonDatasetProfile().setDatasetPathsList(Lists.<DatasetPathUI>newArrayList())); tableDatasetProfileMap.put(tableName, tableDatasetProfile); } setCommonDatasetProfile(tableDatasetProfile.getDatasetProfile(), operatorProfile, majorFragment, inputBytes, inputRecords, PathUtils.parseFullPath(tableName)); tableDatasetProfile.setPushdownQuery(null); } }
private static long findMetric(OperatorProfile operatorProfile, final int id) { return FluentIterable.from(operatorProfile.getMetricList()).firstMatch(new Predicate<MetricValue>() { @Override public boolean apply(@Nullable MetricValue input) { return input.getMetricId() == id; } }).get().getLongValue(); }
public Builder mergeFrom(com.dremio.exec.proto.UserBitShared.OperatorProfile other) { if (other == com.dremio.exec.proto.UserBitShared.OperatorProfile.getDefaultInstance()) return this; if (inputProfileBuilder_ == null) { if (!other.inputProfile_.isEmpty()) { if (other.hasOperatorId()) { setOperatorId(other.getOperatorId()); if (other.hasOperatorType()) { setOperatorType(other.getOperatorType()); if (other.hasSetupNanos()) { setSetupNanos(other.getSetupNanos()); if (other.hasProcessNanos()) { setProcessNanos(other.getProcessNanos()); if (other.hasPeakLocalMemoryAllocated()) { setPeakLocalMemoryAllocated(other.getPeakLocalMemoryAllocated()); if (other.hasWaitNanos()) { setWaitNanos(other.getWaitNanos()); this.mergeUnknownFields(other.getUnknownFields()); return this;
totalTimeInMillis += toMillis(operatorProfile.getProcessNanos() + operatorProfile.getWaitNanos() + operatorProfile.getSetupNanos()); final CoreOperatorType operatorType = CoreOperatorType.valueOf(operatorProfile.getOperatorType()); logger.error("QueryProfile has unknown operator type " + operatorProfile.getOperatorType()); setOperationStats(OperationType.Misc, toMillis(operatorProfile.getProcessNanos())); continue; setWaitInClient(toMillis(operatorProfile.getWaitNanos())); toMillis(operatorProfile.getProcessNanos() + operatorProfile.getWaitNanos() + operatorProfile.getSetupNanos())); break; setOperationStats(OperationType.Reading, toMillis(operatorProfile.getProcessNanos() + operatorProfile.getSetupNanos())); break; setOperationStats(OperationType.Reading, toMillis(operatorProfile.getProcessNanos() + operatorProfile.getSetupNanos())); break; setOperationStats(OperationType.Sort, toMillis(operatorProfile.getProcessNanos() + operatorProfile.getSetupNanos())); break; setOperationStats(OperationType.Sort, toMillis(operatorProfile.getProcessNanos() + operatorProfile.getSetupNanos())); break; setOperationStats(OperationType.Aggregate, toMillis(operatorProfile.getProcessNanos() + operatorProfile.getSetupNanos())); break; case STREAMING_AGGREGATE: setOperationStats(OperationType.Aggregate, toMillis(operatorProfile.getProcessNanos() + operatorProfile.getSetupNanos())); break;
long incomingRecords = 0; long batches = 0; for (final StreamProfile sp : op.getInputProfileList()) { incomingRecords += sp.getRecords(); batches += sp.getBatches(); long totalNanos = 0; for (OperatorProfile op : operators) { totalNanos += op.getSetupNanos(); totalNanos += op.getProcessNanos(); totalNanos += op.getWaitNanos();
break; case 4: builder.addOperatorProfile(input.mergeObject(com.dremio.exec.proto.UserBitShared.OperatorProfile.newBuilder(), com.dremio.exec.proto.SchemaUserBitShared.OperatorProfile.MERGE));
final boolean toSkip = isHashAgg && renderingOldProfiles(op); final Number[] values = new Number[metricIds.length]; for (final MetricValue metric : op.getMetricList()) { int metricId = metric.getMetricId(); if (toSkip) {
major.getMajorFragmentId(), op.getOperatorId()); if (!opmap.containsKey(ip)) { final List<ImmutablePair<OperatorProfile, Integer>> l = new ArrayList<>();
assertEquals(1L, scanProfile.get().getInputProfile(0).getRecords());
assertTrue("Expected wait time is non-zero, but got zero wait time", operatorProfile.getWaitNanos() > 0);
private void setCommonDatasetProfile(CommonDatasetProfile datasetProfile, OperatorProfile operatorProfile, MajorFragmentProfile majorFragment, long inputBytes, long inputRecords, List<String> datasetPath) { if (datasetProfile.getBytesRead() != null) { datasetProfile.setBytesRead(datasetProfile.getRecordsRead() + inputBytes); } else { datasetProfile.setBytesRead(inputBytes); } if (datasetProfile.getRecordsRead() != null) { datasetProfile.setRecordsRead(datasetProfile.getRecordsRead() + inputRecords); } else { datasetProfile.setRecordsRead(inputRecords); } final DatasetPathUI datasetPathUI = new DatasetPathUI().setDatasetPathList(datasetPath); if (!datasetProfile.getDatasetPathsList().contains(datasetPathUI)) { datasetProfile.getDatasetPathsList().add(datasetPathUI); } if (datasetProfile.getWaitOnSource() != null) { datasetProfile.setWaitOnSource(datasetProfile.getWaitOnSource() + toMillis(operatorProfile.getWaitNanos())); } else { datasetProfile.setWaitOnSource(toMillis(operatorProfile.getWaitNanos())); } if (datasetProfile.getParallelism() != null) { datasetProfile.setParallelism(datasetProfile.getParallelism() + 1); } else { datasetProfile.setParallelism(1); } }
@Test public void testReadIOStats() throws Exception { FileSystemWrapper dfs = null; InputStream is = null; Configuration conf = new Configuration(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); OpProfileDef profileDef = new OpProfileDef(0 /*operatorId*/, 0 /*operatorType*/, 0 /*inputCount*/); OperatorStats stats = new OperatorStats(profileDef, null /*allocator*/); // start wait time method in OperatorStats expects the OperatorStats state to be in "processing" stats.startProcessing(); try { dfs = new FileSystemWrapper(conf, stats, null); is = dfs.open(new Path(tempFilePath)); byte[] buf = new byte[8000]; while (is.read(buf, 0, buf.length) != -1) { } } finally { stats.stopProcessing(); if (is != null) { is.close(); } if (dfs != null) { dfs.close(); } } OperatorProfile operatorProfile = stats.getProfile(); assertTrue("Expected wait time is non-zero, but got zero wait time", operatorProfile.getWaitNanos() > 0); }
for (ImmutablePair<OperatorProfile, Integer> ip : ops) { OperatorProfile profile = ip.getLeft(); setupSum += profile.getSetupNanos(); processSum += profile.getProcessNanos(); waitSum += profile.getWaitNanos(); memSum += profile.getPeakLocalMemoryAllocated(); tb.appendNanos(shortSetup.getLeft().getSetupNanos()); tb.appendNanos(Math.round(setupSum / size)); tb.appendNanos(longSetup.getLeft().getSetupNanos()); tb.appendNanos(shortProcess.getLeft().getProcessNanos()); tb.appendNanos(Math.round(processSum / size)); tb.appendNanos(longProcess.getLeft().getProcessNanos()); tb.appendNanos(shortWait.getLeft().getWaitNanos()); tb.appendNanos(Math.round(waitSum / size)); tb.appendNanos(longWait.getLeft().getWaitNanos()); tb.appendBytes(peakMem.getLeft().getPeakLocalMemoryAllocated(), null);
@Test public void testGetMetricsTableHandlesNotRegisteredMetrics() { OperatorProfile op = OperatorProfile .newBuilder().addMetric( UserBitShared.MetricValue.newBuilder() .setMetricId(1) .setDoubleValue(200)) .addMetric( UserBitShared.MetricValue.newBuilder() .setMetricId(3) .setDoubleValue(21)) .setOperatorId(UserBitShared.CoreOperatorType.PARQUET_ROW_GROUP_SCAN.getNumber()) .build(); ImmutablePair<OperatorProfile, Integer> pair = new ImmutablePair<>(op, 1); OperatorWrapper ow = new OperatorWrapper(1, ImmutableList.of(pair), OperatorMetricRegistry.getCoreOperatorTypeMetricsMap()); String html = ow.getMetricsTable(); Pattern columnPattern = Pattern.compile("<td>.*?</td>"); Matcher matcher = columnPattern.matcher(html); int count = 0; while (matcher.find()){ count++; } // Unregistered metrics should not appear on profiles screen assertEquals(2, count); }
public String getContent() { TableBuilder builder = new TableBuilder(OPERATOR_COLUMNS); for (ImmutablePair<OperatorProfile, Integer> ip : ops) { int minor = ip.getRight(); OperatorProfile op = ip.getLeft(); String path = new OperatorPathBuilder().setMajor(major).setMinor(minor).setOperator(op).build(); builder.appendCell(path, null); builder.appendNanos(op.getSetupNanos()); builder.appendNanos(op.getProcessNanos()); builder.appendNanos(op.getWaitNanos()); long maxBatches = Long.MIN_VALUE; long maxRecords = Long.MIN_VALUE; for (StreamProfile sp : op.getInputProfileList()) { maxBatches = Math.max(sp.getBatches(), maxBatches); maxRecords = Math.max(sp.getRecords(), maxRecords); } builder.appendFormattedInteger(maxBatches, null); builder.appendFormattedInteger(maxRecords, null); builder.appendBytes(op.getPeakLocalMemoryAllocated(), null); } return builder.build(); }
public Builder toBuilder() { return newBuilder(this); }
public void writeTo(io.protostuff.Output output, com.dremio.exec.proto.UserBitShared.OperatorProfile message) throws java.io.IOException { for(com.dremio.exec.proto.UserBitShared.StreamProfile inputProfile : message.getInputProfileList()) output.writeObject(1, inputProfile, com.dremio.exec.proto.SchemaUserBitShared.StreamProfile.WRITE, true); if(message.hasOperatorId()) output.writeInt32(3, message.getOperatorId(), false); if(message.hasOperatorType()) output.writeInt32(4, message.getOperatorType(), false); if(message.hasSetupNanos()) output.writeInt64(5, message.getSetupNanos(), false); if(message.hasProcessNanos()) output.writeInt64(6, message.getProcessNanos(), false); if(message.hasPeakLocalMemoryAllocated()) output.writeInt64(7, message.getPeakLocalMemoryAllocated(), false); for(com.dremio.exec.proto.UserBitShared.MetricValue metric : message.getMetricList()) output.writeObject(8, metric, com.dremio.exec.proto.SchemaUserBitShared.MetricValue.WRITE, true); if(message.hasWaitNanos()) output.writeInt64(9, message.getWaitNanos(), false); } public boolean isInitialized(com.dremio.exec.proto.UserBitShared.OperatorProfile message)
private boolean renderingOldProfiles(OperatorProfile op) { final List<MetricValue> metricValues = op.getMetricList(); for (MetricValue metric : metricValues) { final int metricId = metric.getMetricId(); if (metricId == HashAggStats.SKIP_METRIC_START) { /* if the ordinal (metric id) to skip is indeed present * in the serialized profile that we are trying to render * then we are definitely working with new profiles */ return false; } } return true; } }
public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder(com.dremio.exec.proto.UserBitShared.OperatorProfile prototype) {
public String getId() { return String.format("operator-%d-%d", major, ops.get(0).getLeft().getOperatorId()); }