private boolean madeProgress(final OperatorProfile prev, final OperatorProfile cur) { return prev.getInputProfileCount() != cur.getInputProfileCount() || !prev.getInputProfileList().equals(cur.getInputProfileList()) || prev.getMetricCount() != cur.getMetricCount() || !prev.getMetricList().equals(cur.getMetricList()); }
private long getRowsProcessed(MinorFragmentProfile profile) { long maxRecords = 0; for (OperatorProfile operatorProfile : profile.getOperatorProfileList()) { long records = 0; for (StreamProfile inputProfile :operatorProfile.getInputProfileList()) { if (inputProfile.hasRecords()) { records += inputProfile.getRecords(); } } maxRecords = Math.max(maxRecords, records); } return maxRecords; }
private void setScanStats(CoreOperatorType operatorType, OperatorProfile operatorProfile, MajorFragmentProfile majorFragment) { final String operatorId = createOperatorKey(majorFragment.getMajorFragmentId(), operatorProfile.getOperatorId()); long inputBytes = 0L; long inputRecords = 0L; if (operatorProfile.getInputProfileList() != null) { for (StreamProfile stream : operatorProfile.getInputProfileList()) { inputRecords += stream.getRecords(); inputBytes += stream.getSize(); } addInputBytesAndRecords(inputBytes, inputRecords); } if (operatorToTable.containsKey(operatorId)) { // TODO check if its fs based or table based using operatorType final String tableName = operatorToTable.get(operatorId); TableDatasetProfile tableDatasetProfile = tableDatasetProfileMap.get(tableName); if (tableDatasetProfile == null) { tableDatasetProfile = new TableDatasetProfile(); tableDatasetProfile.setDatasetProfile(new CommonDatasetProfile().setDatasetPathsList(Lists.<DatasetPathUI>newArrayList())); tableDatasetProfileMap.put(tableName, tableDatasetProfile); } setCommonDatasetProfile(tableDatasetProfile.getDatasetProfile(), operatorProfile, majorFragment, inputBytes, inputRecords, PathUtils.parseFullPath(tableName)); tableDatasetProfile.setPushdownQuery(null); } }
case TEXT_WRITER: case JSON_WRITER: setOutputStats(operatorProfile.getInputProfileList()); setOperationStats(OperationType.Writing, toMillis(operatorProfile.getProcessNanos() + operatorProfile.getSetupNanos())); break; List<StreamProfile> streams = profile.getInputProfileList(); setOutputStatsForQueryResults(streams, outputLimited);
long incomingRecords = 0; long batches = 0; for (final StreamProfile sp : op.getInputProfileList()) { incomingRecords += sp.getRecords(); batches += sp.getBatches();
public void writeTo(io.protostuff.Output output, com.dremio.exec.proto.UserBitShared.OperatorProfile message) throws java.io.IOException { for(com.dremio.exec.proto.UserBitShared.StreamProfile inputProfile : message.getInputProfileList()) output.writeObject(1, inputProfile, com.dremio.exec.proto.SchemaUserBitShared.StreamProfile.WRITE, true); if(message.hasOperatorId()) output.writeInt32(3, message.getOperatorId(), false); if(message.hasOperatorType()) output.writeInt32(4, message.getOperatorType(), false); if(message.hasSetupNanos()) output.writeInt64(5, message.getSetupNanos(), false); if(message.hasProcessNanos()) output.writeInt64(6, message.getProcessNanos(), false); if(message.hasPeakLocalMemoryAllocated()) output.writeInt64(7, message.getPeakLocalMemoryAllocated(), false); for(com.dremio.exec.proto.UserBitShared.MetricValue metric : message.getMetricList()) output.writeObject(8, metric, com.dremio.exec.proto.SchemaUserBitShared.MetricValue.WRITE, true); if(message.hasWaitNanos()) output.writeInt64(9, message.getWaitNanos(), false); } public boolean isInitialized(com.dremio.exec.proto.UserBitShared.OperatorProfile message)
public String getContent() { TableBuilder builder = new TableBuilder(OPERATOR_COLUMNS); for (ImmutablePair<OperatorProfile, Integer> ip : ops) { int minor = ip.getRight(); OperatorProfile op = ip.getLeft(); String path = new OperatorPathBuilder().setMajor(major).setMinor(minor).setOperator(op).build(); builder.appendCell(path, null); builder.appendNanos(op.getSetupNanos()); builder.appendNanos(op.getProcessNanos()); builder.appendNanos(op.getWaitNanos()); long maxBatches = Long.MIN_VALUE; long maxRecords = Long.MIN_VALUE; for (StreamProfile sp : op.getInputProfileList()) { maxBatches = Math.max(sp.getBatches(), maxBatches); maxRecords = Math.max(sp.getRecords(), maxRecords); } builder.appendFormattedInteger(maxBatches, null); builder.appendFormattedInteger(maxRecords, null); builder.appendBytes(op.getPeakLocalMemoryAllocated(), null); } return builder.build(); }