public Builder mergeFrom(org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile other) { if (other == org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile.getDefaultInstance()) return this; if (other.hasState()) { setState(other.getState()); if (other.hasError()) { mergeError(other.getError()); if (other.hasMinorFragmentId()) { setMinorFragmentId(other.getMinorFragmentId()); if (other.hasStartTime()) { setStartTime(other.getStartTime()); if (other.hasEndTime()) { setEndTime(other.getEndTime()); if (other.hasMemoryUsed()) { setMemoryUsed(other.getMemoryUsed()); if (other.hasMaxMemoryUsed()) { setMaxMemoryUsed(other.getMaxMemoryUsed()); if (other.hasEndpoint()) { mergeEndpoint(other.getEndpoint()); if (other.hasLastUpdate()) { setLastUpdate(other.getLastUpdate()); if (other.hasLastProgress()) {
Collections.sort(minors, Comparators.minorId); for (final MinorFragmentProfile minor : minors) { String fragmentHostName = minor.getEndpoint().getAddress(); final List<OperatorProfile> ops = new ArrayList<>(minor.getOperatorProfileList()); Collections.sort(ops, Comparators.operatorId); new ImmutablePair<>(op, minor.getMinorFragmentId()), fragmentHostName));
long minorFragmentStart = fragmentWrapper.getStartTime(); if (minorFragmentStart > 0 && minorFragmentStart < estimatedPlanEnd) { estimatedPlanEnd = minorFragmentStart;
public Builder mergeFrom(org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile other) { if (other == org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile.getDefaultInstance()) return this; if (other.hasState()) { setState(other.getState()); if (other.hasError()) { mergeError(other.getError()); if (other.hasMinorFragmentId()) { setMinorFragmentId(other.getMinorFragmentId()); if (other.hasStartTime()) { setStartTime(other.getStartTime()); if (other.hasEndTime()) { setEndTime(other.getEndTime()); if (other.hasMemoryUsed()) { setMemoryUsed(other.getMemoryUsed()); if (other.hasMaxMemoryUsed()) { setMaxMemoryUsed(other.getMaxMemoryUsed()); if (other.hasEndpoint()) { mergeEndpoint(other.getEndpoint()); if (other.hasLastUpdate()) { setLastUpdate(other.getLastUpdate()); if (other.hasLastProgress()) {
tb.appendMillis(firstStart.getStartTime() - start); tb.appendMillis(lastStart.getStartTime() - start); tb.appendMillis(firstEnd.getEndTime() - start); tb.appendMillis(lastEnd.getEndTime() - start); long cumulativeWaitInNanos = 0L; for (final MinorFragmentProfile p : complete) { cumulativeFragmentDurationInMillis += p.getEndTime() - p.getStartTime(); List<OperatorProfile> opProfileList = p.getOperatorProfileList(); for (OperatorProfile operatorProfile : opProfileList) { cumulativeProcessInNanos += operatorProfile.getProcessNanos(); tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime()); tb.appendMillis(cumulativeFragmentDurationInMillis / complete.size()); tb.appendMillis(longRun.getEndTime() - longRun.getStartTime()); tb.appendMillis(System.currentTimeMillis()-lastUpdate.getLastUpdate()); tb.appendMillis(System.currentTimeMillis()-lastProgress.getLastProgress()); tb.appendBytes(maxMem.getMaxMemoryUsed());
tb.appendMillis(firstStart.getStartTime() - start); tb.appendMillis(lastStart.getStartTime() - start); tb.appendMillis(firstEnd.getEndTime() - start); tb.appendMillis(lastEnd.getEndTime() - start); double totalWaitInMillis = 0.0d; for (final MinorFragmentProfile p : complete) { totalDuration += p.getEndTime() - p.getStartTime(); List<OperatorProfile> opProfileList = p.getOperatorProfileList(); for (OperatorProfile operatorProfile : opProfileList) { totalProcessInMillis += operatorProfile.getProcessNanos()/1E6; tb.appendMillis(shortRun.getEndTime() - shortRun.getStartTime()); tb.appendMillis(totalDuration / complete.size()); tb.appendMillis(longRun.getEndTime() - longRun.getStartTime()); tb.appendBytes(maxMem.getMaxMemoryUsed());
final ArrayList<OperatorProfile> ops = new ArrayList<>(minor.getOperatorProfileList()); attributeMap.put("data-order", String.valueOf(minor.getMinorFragmentId())); //Overwrite values from previous fragments builder.appendCell(new OperatorPathBuilder().setMajor(major).setMinor(minor).build(), attributeMap); builder.appendCell(minor.getEndpoint().getAddress()); builder.appendMillis(minor.getStartTime() - start); builder.appendMillis(minor.getEndTime() - start); builder.appendMillis(minor.getEndTime() - minor.getStartTime()); builder.appendTime(minor.getLastUpdate()); builder.appendTime(minor.getLastProgress()); builder.appendBytes(minor.getMaxMemoryUsed()); builder.appendCell(minor.getState().name()); builder.appendCell(major.getMajorFragmentId() + "-" + m.getMinorFragmentId()); builder.appendRepeated(m.getState().toString(), null, NUM_NULLABLE_FRAGMENTS_COLUMNS);
public String getPlanningDuration() { //Check if Planning End is known if (profile.getPlanEnd() > 0L) { return (new SimpleDurationFormat(profile.getStart(), profile.getPlanEnd())).verbose(); } //Check if any fragments have started if (profile.getFragmentProfileCount() > 0) { //Init Planning End Time long estimatedPlanEnd = Long.MAX_VALUE; //Using Screen MajorFragment as reference MajorFragmentProfile majorFrag0 = profile.getFragmentProfile(0); //Searching for earliest starting fragment for (MinorFragmentProfile fragmentWrapper : majorFrag0.getMinorFragmentProfileList()) { long minorFragmentStart = fragmentWrapper.getStartTime(); if (minorFragmentStart > 0 && minorFragmentStart < estimatedPlanEnd) { estimatedPlanEnd = minorFragmentStart; } } //Provide estimated plan time return (new SimpleDurationFormat(profile.getStart(), estimatedPlanEnd)).verbose() + ESTIMATED_LABEL; } //Unable to estimate/calculate Specific Time spent in Planning return NOT_AVAILABLE_LABEL; }
public void writeTo(com.dyuproject.protostuff.Output output, org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile message) throws java.io.IOException { if(message.hasState()) output.writeEnum(1, message.getState().getNumber(), false); if(message.hasError()) output.writeObject(2, message.getError(), org.apache.drill.exec.proto.SchemaUserBitShared.DrillPBError.WRITE, false); if(message.hasMinorFragmentId()) output.writeInt32(3, message.getMinorFragmentId(), false); for(org.apache.drill.exec.proto.UserBitShared.OperatorProfile operatorProfile : message.getOperatorProfileList()) output.writeObject(4, operatorProfile, org.apache.drill.exec.proto.SchemaUserBitShared.OperatorProfile.WRITE, true); if(message.hasStartTime()) output.writeInt64(5, message.getStartTime(), false); if(message.hasEndTime()) output.writeInt64(6, message.getEndTime(), false); if(message.hasMemoryUsed()) output.writeInt64(7, message.getMemoryUsed(), false); if(message.hasMaxMemoryUsed()) output.writeInt64(8, message.getMaxMemoryUsed(), false); if(message.hasEndpoint()) output.writeObject(9, message.getEndpoint(), org.apache.drill.exec.proto.SchemaCoordinationProtos.DrillbitEndpoint.WRITE, false); if(message.hasLastUpdate()) output.writeInt64(10, message.getLastUpdate(), false); if(message.hasLastProgress()) output.writeInt64(11, message.getLastProgress(), false); } public boolean isInitialized(org.apache.drill.exec.proto.UserBitShared.MinorFragmentProfile message)
@Override public void statusUpdate(final FragmentStatus status) { logger.debug("New fragment status was provided to QueryManager of {}", status); switch(status.getProfile().getState()) { case AWAITING_ALLOCATION: case RUNNING: case CANCELLATION_REQUESTED: updateFragmentStatus(status); break; case FAILED: foreman.addToEventQueue(QueryState.FAILED, new UserRemoteException(status.getProfile().getError())); // fall-through. case FINISHED: case CANCELLED: fragmentDone(status); break; default: throw new UnsupportedOperationException(String.format("Received status of %s", status)); } } };
/** * Sends status to remote Foreman node using Control Tunnel or to Local Foreman bypassing * Control Tunnel and using WorkEventBus. * * @param status */ void sendStatus(final FragmentStatus status) { DrillbitEndpoint foremanNode = foremanDrillbit.get(); if (foremanNode == null) { logger.warn("{}: State {} is not reported as {} is closed", QueryIdHelper.getQueryIdentifier(context.getHandle()), status.getProfile().getState(), this); return; } // Send status for both local and remote foreman node via Tunnel. For local there won't be any network connection // created and it will be submitted locally using LocalControlConnectionManager final ControlTunnel tunnel = context.getController().getTunnel(foremanNode); tunnel.sendFragmentStatus(status); }
private boolean updateFragmentStatus(final FragmentStatus fragmentStatus) { final FragmentHandle fragmentHandle = fragmentStatus.getHandle(); final int majorFragmentId = fragmentHandle.getMajorFragmentId(); final int minorFragmentId = fragmentHandle.getMinorFragmentId(); final FragmentData data = fragmentDataMap.get(majorFragmentId).get(minorFragmentId); final FragmentState oldState = data.getState(); final boolean inTerminalState = isTerminal(oldState); final FragmentState currentState = fragmentStatus.getProfile().getState(); if (inTerminalState || (oldState == FragmentState.CANCELLATION_REQUESTED && !isTerminal(currentState))) { // Already in a terminal state, or invalid state transition from CANCELLATION_REQUESTED. This shouldn't happen. logger.warn(String.format("Received status message for fragment %s after fragment was in state %s. New state was %s", QueryIdHelper.getQueryIdentifier(fragmentHandle), oldState, currentState)); return false; } data.setStatus(fragmentStatus); return oldState != currentState; }
private boolean madeProgress(final FragmentStatus prev, final FragmentStatus cur) { final MinorFragmentProfile previous = prev.getProfile(); final MinorFragmentProfile current = cur.getProfile(); if (previous.getState() != current.getState()) { return true; } if (previous.getOperatorProfileCount() != current.getOperatorProfileCount()) { return true; } for(int i =0; i < current.getOperatorProfileCount(); i++){ if (madeProgress(previous.getOperatorProfile(i), current.getOperatorProfile(i))) { return true; } } return false; }
private long tallyMajorFragmentCost(List<MajorFragmentProfile> majorFragments) { long globalProcessNanos = 0L; for (MajorFragmentProfile majorFP : majorFragments) { String majorFragmentId = new OperatorPathBuilder().setMajor(majorFP).build(); long processNanos = 0L; for (MinorFragmentProfile minorFP : majorFP.getMinorFragmentProfileList()) { for (OperatorProfile op : minorFP.getOperatorProfileList()) { processNanos += op.getProcessNanos(); } } majorFragmentTallyMap.put(majorFragmentId, processNanos); globalProcessNanos += processNanos; } return globalProcessNanos; }
public FragmentState getState() { return status.getProfile().getState(); }
@Override public int compare(final MinorFragmentProfile o1, final MinorFragmentProfile o2) { return Long.compare(o1.getStartTime(), o2.getStartTime()); } };
@Override public int compare(final MinorFragmentProfile o1, final MinorFragmentProfile o2) { return Long.compare(o1.getMaxMemoryUsed(), o2.getMaxMemoryUsed()); } };
@Override public int compare(final MinorFragmentProfile o1, final MinorFragmentProfile o2) { return Long.compare(o1.getLastUpdate(), o2.getLastUpdate()); } };
public OperatorPathBuilder setMinor(MinorFragmentProfile minor) { if (minor != null) { return setMinor(minor.getMinorFragmentId()); } return this; }
@Override public int compare(final MinorFragmentProfile o1, final MinorFragmentProfile o2) { return Long.compare(o1.getMinorFragmentId(), o2.getMinorFragmentId()); } };