vertexProgressMap = status.getVertexProgress(); List<String> vertexNames = vertexProgressMap.keySet() .stream() if (wmContext != null) { Set<String> desiredCounters = wmContext.getSubscribedCounters(); TezCounters dagCounters = status.getDAGCounters(); if (dagCounters != null && desiredCounters != null && !desiredCounters.isEmpty()) { Map<String, Long> currentCounters = getCounterValues(dagCounters, vertexNames, vertexProgressMap, DAGStatus.State state = status.getState(); for (String diag : status.getDiagnostics()) { console.printError(diag); diagnostics.append(diag);
vertexProgressMap = status.getVertexProgress(); DAGStatus.State state = status.getState(); if (done) { if (rc != 0 && status != null) { for (String diag : status.getDiagnostics()) { console.printError(diag); diagnostics.append(diag);
protected void dumpDebugInfo() { DAGStatus dagStatus = getDagStatus(); if( dagStatus == null ) return; flowStep.logWarn( "Tez DAG " + dagId + " state at " + dagStatus.getState() ); flowStep.logWarn( "failure info: " ); for( String diagLine : dagStatus.getDiagnostics() ) flowStep.logWarn( diagLine ); }
@Override public String executionStatus() { return this.status.getState().name(); }
if (dagStatus.getState() != DAGStatus.State.SUCCEEDED) { LOG.info("DAG diagnostics: " + dagStatus.getDiagnostics()); return -1; } else { dagStatus = dagClient.getDAGStatus(Sets.newHashSet(StatusGetOpts.GET_COUNTERS)); TezCounter counter = dagStatus.getDAGCounters().findCounter(COUNTER_GROUP_NAME, MISSING_KEY_COUNTER_NAME); if (counter == null) {
dagStatus = getDAGStatus(statusGetOpts, SLEEP_FOR_COMPLETION); if (!initPrinted && (dagStatus.getState() == DAGStatus.State.INITING || dagStatus.getState() == DAGStatus.State.SUBMITTED)) { initPrinted = true; // Print once log("Waiting for DAG to start running"); if (dagStatus.getState() == DAGStatus.State.RUNNING || dagStatus.getState() == DAGStatus.State.SUCCEEDED || dagStatus.getState() == DAGStatus.State.FAILED || dagStatus.getState() == DAGStatus.State.KILLED || dagStatus.getState() == DAGStatus.State.ERROR) { break; while (!dagStatus.isCompleted()) { if (!runningPrinted) { log("DAG initialized: CurrentState=Running"); vertexNames = getDAGStatus(statusGetOpts).getVertexProgress().keySet(); log("DAG completed. " + "FinalState=" + dagStatus.getState()); return dagStatus;
DAGStatus dagStatus = dagClient.getDAGStatus( (displayDAGCounters ? opts : null)); Progress progress = dagStatus.getDAGProgress(); double vProgressFloat = 0.0f; if (progress != null) { System.out.println(""); System.out.println("DAG: State: " + dagStatus.getState() + " Progress: " + (progress.getTotalTaskCount() < 0 ? formatter.format(0.0f) : TezCounters counters = dagStatus.getDAGCounters(); if (counters != null) { System.out.println("DAG Counters: " + counters);
private TezCounters hiveCounters(DAGClient dagClient) { try { return dagClient.getDAGStatus(EnumSet.of(StatusGetOpts.GET_COUNTERS)).getDAGCounters(); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; }
@Override public synchronized float getMapProgress() { if(dagStatus.getVertexProgress() != null) { return getProgress(MultiStageMRConfigUtil.getInitialMapVertexName()); } if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) { return 1.0f; } return 0.0f; }
private void printDAGStatus(Set<String> vertexNames, Set<StatusGetOpts> opts, DAGStatus dagStatus, Progress dagProgress) throws IOException, TezException { double vProgressFloat = 0.0f; log("DAG: State: " + dagStatus.getState() + " Progress: " + formatter.format(getProgress(dagProgress)) + " " + dagProgress); boolean displayCounter = opts != null && opts.contains(StatusGetOpts.GET_COUNTERS); if (displayCounter) { TezCounters counters = dagStatus.getDAGCounters(); if (counters != null) { log("DAG Counters:\n" + counters);
if (dagStatus.getState() == DAGStatus.State.INITING || dagStatus.getState() == DAGStatus.State.SUBMITTED) { long timeoutAbsolute = startTime + timeout; while (timeout < 0 if (dagStatus.getState() == DAGStatus.State.RUNNING) { if (dagStatus.getSource() == DagStatusSource.AM) { } else if(dagStatus.getState() == DAGStatus.State.SUCCEEDED || dagStatus.getState() == DAGStatus.State.FAILED || dagStatus.getState() == DAGStatus.State.KILLED || dagStatus.getState() == DAGStatus.State.ERROR) { if (dagStatus.getSource() == DagStatusSource.RM) { return getDAGStatusInternal(statusOptions, 0); } else {
if (dagStatus.isCompleted()) { try { dagStatus = dagClient.getDAGStatus(statusGetOpts); dagCounters = dagStatus.getDAGCounters(); TezSessionManager.freeSession(tezClient); try {
@Override public synchronized String getFailureInfo() { return Joiner.on(". ").join(dagStatus.getDiagnostics()); }
public Map<String, Float> getVertexProgress() { Map<String, Float> vertexProgress = Maps.newHashMap(); for (Map.Entry<String, Progress> entry : dagStatus.getVertexProgress().entrySet()) { Progress p = entry.getValue(); float progress = (float)p.getSucceededTaskCount() / (float)p.getTotalTaskCount(); vertexProgress.put(entry.getKey(), progress); } return vertexProgress; }
private boolean isDagStatusComplete() { DAGStatus dagStatus = getDagStatus(); if( dagStatus == null ) flowStep.logWarn( "getDagStatus returned null" ); return dagStatus != null && dagStatus.isCompleted(); }
public float getDAGProgress() { Progress p = dagStatus.getDAGProgress(); return p == null ? 0 : (float)p.getSucceededTaskCount() / (float)p.getTotalTaskCount(); }
protected void dumpDebugInfo() { DAGStatus dagStatus = getDagStatus(); if( dagStatus == null ) return; flowStep.logWarn( "Tez DAG " + dagId + " state at " + dagStatus.getState() ); flowStep.logWarn( "failure info: " ); for( String diagLine : dagStatus.getDiagnostics() ) flowStep.logWarn( diagLine ); }
@Override public String executionStatus() { return this.status.getState().name(); }
Map<String, Progress> progressMap = status.getVertexProgress(); DAGStatus.State state = status.getState(); heartbeater.heartbeat(); if (done) { if (rc != 0 && status != null) { for (String diag : status.getDiagnostics()) { console.printError(diag); diagnostics.append(diag);
private TezCounters hiveCounters(DAGClient dagClient) { try { return dagClient.getDAGStatus(EnumSet.of(StatusGetOpts.GET_COUNTERS)).getDAGCounters(); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; }