public Builder mergeFrom(org.apache.drill.exec.proto.UserBitShared.OperatorProfile other) { if (other == org.apache.drill.exec.proto.UserBitShared.OperatorProfile.getDefaultInstance()) return this; if (inputProfileBuilder_ == null) { if (!other.inputProfile_.isEmpty()) { if (other.hasOperatorId()) { setOperatorId(other.getOperatorId()); if (other.hasOperatorType()) { setOperatorType(other.getOperatorType()); if (other.hasSetupNanos()) { setSetupNanos(other.getSetupNanos()); if (other.hasProcessNanos()) { setProcessNanos(other.getProcessNanos()); if (other.hasPeakLocalMemoryAllocated()) { setPeakLocalMemoryAllocated(other.getPeakLocalMemoryAllocated()); if (other.hasWaitNanos()) { setWaitNanos(other.getWaitNanos()); this.mergeUnknownFields(other.getUnknownFields()); return this;
public org.apache.drill.exec.proto.UserBitShared.OperatorProfile buildPartial() { org.apache.drill.exec.proto.UserBitShared.OperatorProfile result = new org.apache.drill.exec.proto.UserBitShared.OperatorProfile(this); int from_bitField0_ = bitField0_; int to_bitField0_ = 0;
public Builder mergeFrom(org.apache.drill.exec.proto.UserBitShared.OperatorProfile other) { if (other == org.apache.drill.exec.proto.UserBitShared.OperatorProfile.getDefaultInstance()) return this; if (inputProfileBuilder_ == null) { if (!other.inputProfile_.isEmpty()) { if (other.hasOperatorId()) { setOperatorId(other.getOperatorId()); if (other.hasOperatorType()) { setOperatorType(other.getOperatorType()); if (other.hasSetupNanos()) { setSetupNanos(other.getSetupNanos()); if (other.hasProcessNanos()) { setProcessNanos(other.getProcessNanos()); if (other.hasPeakLocalMemoryAllocated()) { setPeakLocalMemoryAllocated(other.getPeakLocalMemoryAllocated()); if (other.hasWaitNanos()) { setWaitNanos(other.getWaitNanos()); this.mergeUnknownFields(other.getUnknownFields()); return this;
setupSum += profile.getSetupNanos(); processSum += profile.getProcessNanos(); waitSum += profile.getWaitNanos(); memSum += profile.getPeakLocalMemoryAllocated(); for (final StreamProfile sp : profile.getInputProfileList()) { recordSum += sp.getRecords(); for (MetricValue metricVal : profile.getMetricList()) { if (metricVal.getMetricId() == spillCycleMetricIndex) { long spillCycles = metricVal.getLongValue(); tb.appendNanos(longSetup.getLeft().getSetupNanos()); tb.appendNanos(longProcess.getLeft().getProcessNanos()); tb.appendNanos(shortWait.getLeft().getWaitNanos()); tb.appendNanos(Math.round(waitSum / size)); tb.appendNanos(longWait.getLeft().getWaitNanos()); tb.appendBytes(peakMem.getLeft().getPeakLocalMemoryAllocated(), maxSpillMap);
public org.apache.drill.exec.proto.UserBitShared.OperatorProfile buildPartial() { org.apache.drill.exec.proto.UserBitShared.OperatorProfile result = new org.apache.drill.exec.proto.UserBitShared.OperatorProfile(this); int from_bitField0_ = bitField0_; int to_bitField0_ = 0;
major.getMajorFragmentId(), op.getOperatorId()); if (!opmap.containsKey(ip)) { final List<ImmutablePair<ImmutablePair<OperatorProfile, Integer>, String>> l = new ArrayList<>();
cumulativeProcessInNanos += operatorProfile.getProcessNanos(); cumulativeWaitInNanos += operatorProfile.getWaitNanos();
for (final MetricValue metric : op.getMetricList()) { if (metric.getMetricId() < metricNames.length) { if (metric.hasLongValue()) {
break; case 4: builder.addOperatorProfile(input.mergeObject(org.apache.drill.exec.proto.UserBitShared.OperatorProfile.newBuilder(), org.apache.drill.exec.proto.SchemaUserBitShared.OperatorProfile.MERGE));
totalProcessInMillis += operatorProfile.getProcessNanos()/1E6; totalWaitInMillis += operatorProfile.getWaitNanos()/1E6;
long incomingRecords = 0; long batches = 0; for (final StreamProfile sp : op.getInputProfileList()) { incomingRecords += sp.getRecords(); batches += sp.getBatches();
public OperatorWrapper(int major, List<ImmutablePair<ImmutablePair<OperatorProfile, Integer>, String>> opsAndHostsList, Map<String, String> phyOperMap) { Preconditions.checkArgument(opsAndHostsList.size() > 0); this.major = major; firstProfile = opsAndHostsList.get(0).getLeft().getLeft(); operatorType = CoreOperatorType.valueOf(firstProfile.getOperatorType()); //Update Name from Physical Map String path = new OperatorPathBuilder().setMajor(major).setOperator(firstProfile).build(); //Use Plan Extracted Operator Names if available String extractedOpName = phyOperMap.get(path); String inferredOpName = operatorType == null ? UNKNOWN_OPERATOR : operatorType.toString(); //Revert to inferred names for exceptional cases // 1. Extracted 'FLATTEN' operator is NULL // 2. Extracted 'SCAN' could be a PARQUET_ROW_GROUP_SCAN, or KAFKA_SUB_SCAN, or etc. // 3. Extracted 'UNION_EXCHANGE' could be a SINGLE_SENDER or UNORDERED_RECEIVER if (extractedOpName == null || inferredOpName.contains(extractedOpName) || extractedOpName.endsWith("_EXCHANGE")) { operatorName = inferredOpName; } else { operatorName = extractedOpName; } this.opsAndHosts = opsAndHostsList; size = opsAndHostsList.size(); }
public String getContent() { TableBuilder builder = new TableBuilder(OPERATOR_COLUMNS, OPERATOR_COLUMNS_TOOLTIP, true); Map<String, String> attributeMap = new HashMap<String, String>(); //Reusing for different fragments for (ImmutablePair<ImmutablePair<OperatorProfile, Integer>, String> ip : opsAndHosts) { int minor = ip.getLeft().getRight(); OperatorProfile op = ip.getLeft().getLeft(); attributeMap.put("data-order", String.valueOf(minor)); //Overwrite values from previous fragments String path = new OperatorPathBuilder().setMajor(major).setMinor(minor).setOperator(op).build(); builder.appendCell(path, attributeMap); builder.appendCell(ip.getRight()); builder.appendNanos(op.getSetupNanos()); builder.appendNanos(op.getProcessNanos()); builder.appendNanos(op.getWaitNanos()); long maxBatches = Long.MIN_VALUE; long maxRecords = Long.MIN_VALUE; for (StreamProfile sp : op.getInputProfileList()) { maxBatches = Math.max(sp.getBatches(), maxBatches); maxRecords = Math.max(sp.getRecords(), maxRecords); } builder.appendFormattedInteger(maxBatches); builder.appendFormattedInteger(maxRecords); builder.appendBytes(op.getPeakLocalMemoryAllocated()); } return builder.build(); }
public void writeTo(com.dyuproject.protostuff.Output output, org.apache.drill.exec.proto.UserBitShared.OperatorProfile message) throws java.io.IOException { for(org.apache.drill.exec.proto.UserBitShared.StreamProfile inputProfile : message.getInputProfileList()) output.writeObject(1, inputProfile, org.apache.drill.exec.proto.SchemaUserBitShared.StreamProfile.WRITE, true); if(message.hasOperatorId()) output.writeInt32(3, message.getOperatorId(), false); if(message.hasOperatorType()) output.writeInt32(4, message.getOperatorType(), false); if(message.hasSetupNanos()) output.writeInt64(5, message.getSetupNanos(), false); if(message.hasProcessNanos()) output.writeInt64(6, message.getProcessNanos(), false); if(message.hasPeakLocalMemoryAllocated()) output.writeInt64(7, message.getPeakLocalMemoryAllocated(), false); for(org.apache.drill.exec.proto.UserBitShared.MetricValue metric : message.getMetricList()) output.writeObject(8, metric, org.apache.drill.exec.proto.SchemaUserBitShared.MetricValue.WRITE, true); if(message.hasWaitNanos()) output.writeInt64(9, message.getWaitNanos(), false); } public boolean isInitialized(org.apache.drill.exec.proto.UserBitShared.OperatorProfile message)
public Builder newBuilderForType() { return newBuilder(); } public static Builder newBuilder(org.apache.drill.exec.proto.UserBitShared.OperatorProfile prototype) {
private long tallyMajorFragmentCost(List<MajorFragmentProfile> majorFragments) { long globalProcessNanos = 0L; for (MajorFragmentProfile majorFP : majorFragments) { String majorFragmentId = new OperatorPathBuilder().setMajor(majorFP).build(); long processNanos = 0L; for (MinorFragmentProfile minorFP : majorFP.getMinorFragmentProfileList()) { for (OperatorProfile op : minorFP.getOperatorProfileList()) { processNanos += op.getProcessNanos(); } } majorFragmentTallyMap.put(majorFragmentId, processNanos); globalProcessNanos += processNanos; } return globalProcessNanos; }
public String getId() { return String.format("operator-%d-%d", major, opsAndHosts.get(0).getLeft().getLeft().getOperatorId()); }
public OperatorProfile getProfile() { final OperatorProfile.Builder b = OperatorProfile // .newBuilder() // .setOperatorType(operatorType) // .setOperatorId(operatorId) // .setSetupNanos(setupNanos) // .setProcessNanos(processingNanos) .setWaitNanos(waitNanos); if (allocator != null) { b.setPeakLocalMemoryAllocated(allocator.getPeakMemoryAllocation()); } addAllMetrics(b); return b.build(); }
public Builder toBuilder() { return newBuilder(this); }
@Override public int compare(final Pair<OperatorProfile, Integer> o1, final Pair<OperatorProfile, Integer> o2) { return Long.compare(o1.getLeft().getProcessNanos(), o2.getLeft().getProcessNanos()); } };