@Override public VertexStatus getVertexStatus(String vertexName, Set<StatusGetOpts> statusOptions) throws IOException, TezException { synchronized (dagClient) { return dagClient.getVertexStatus(vertexName, statusOptions); } }
private VertexStatus vertexStatus(Set<StatusGetOpts> statusOptions, String vertexName) { try { return dagClient.getVertexStatus(vertexName, statusOptions); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; }
private VertexStatus vertexStatus(Set<StatusGetOpts> statusOptions, String vertexName) { try { return dagClient.getVertexStatus(vertexName, statusOptions); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; }
@Override public VertexStatus getVertexStatus(String vertexName, Set<StatusGetOpts> statusOptions) throws IOException, TezException { synchronized (dagClient) { return dagClient.getVertexStatus(vertexName, statusOptions); } }
private TezCounters vertexCounter(Set<StatusGetOpts> statusOptions, String vertexName) { try { return dagClient.getVertexStatus(vertexName, statusOptions).getVertexCounters(); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; }
private TezCounters vertexCounter(Set<StatusGetOpts> statusOptions, String vertexName) { try { return dagClient.getVertexStatus(vertexName, statusOptions).getVertexCounters(); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; }
private TezCounters vertexCounters(Set<StatusGetOpts> statusOptions, String vertexName) { try { return dagClient.getVertexStatus(vertexName, statusOptions).getVertexCounters(); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; } }
private TezCounters vertexCounters(Set<StatusGetOpts> statusOptions, String vertexName) { try { return dagClient.getVertexStatus(vertexName, statusOptions).getVertexCounters(); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; } }
private TezCounters vertexCounter(Set<StatusGetOpts> statusOptions, String vertexName) { try { return dagClient.getVertexStatus(vertexName, statusOptions).getVertexCounters(); } catch (IOException | TezException e) { // best attempt, shouldn't really kill DAG for this } return null; }
/** * Try to get most the data required from dagClient in the constructor itself so that even after * the tez job has finished this object can be used for later use.s */ TezProgressMonitor(DAGClient dagClient, DAGStatus status, List<BaseWork> topSortedWork, Map<String, Progress> progressMap, SessionState.LogHelper console, long executionStartTime) throws IOException, TezException { this.status = status; this.topSortedWork = topSortedWork; this.console = console; this.executionStartTime = executionStartTime; for (Map.Entry<String, Progress> entry : progressMap.entrySet()) { String vertexName = entry.getKey(); progressCountsMap.put(vertexName, new VertexProgress(entry.getValue(), status.getState())); try { vertexStatusMap.put(vertexName, dagClient.getVertexStatus(vertexName, null)); } catch (IOException e) { // best attempt, shouldn't really kill DAG for this } } }
/** * Try to get most the data required from dagClient in the constructor itself so that even after * the tez job has finished this object can be used for later use.s */ TezProgressMonitor(DAGClient dagClient, DAGStatus status, Map<String, BaseWork> workMap, Map<String, Progress> progressMap, SessionState.LogHelper console, long executionStartTime) throws IOException, TezException { this.status = status; this.workMap = workMap; this.console = console; this.executionStartTime = executionStartTime; for (Map.Entry<String, Progress> entry : progressMap.entrySet()) { String vertexName = entry.getKey(); progressCountsMap.put(vertexName, new VertexProgress(entry.getValue(), status.getState())); try { vertexStatusMap.put(vertexName, dagClient.getVertexStatus(vertexName, null)); } catch (IOException e) { // best attempt, shouldn't really kill DAG for this } } }
@Test public void setupInternalStateOnObjectCreation() throws IOException, TezException { when(dagStatus.getState()).thenReturn(DAGStatus.State.RUNNING); when(dagClient.getVertexStatus(eq(MAPPER), anySet())).thenReturn(succeeded); when(dagClient.getVertexStatus(eq(REDUCER), anySet())).thenReturn(running); TezProgressMonitor monitor = new TezProgressMonitor(dagClient, dagStatus, new ArrayList<BaseWork>(), progressMap(), console, Long.MAX_VALUE); verify(dagClient).getVertexStatus(eq(MAPPER), isNull(Set.class)); verify(dagClient).getVertexStatus(eq(REDUCER), isNull(Set.class)); verifyNoMoreInteractions(dagClient); assertThat(monitor.vertexStatusMap.keySet(), hasItems(MAPPER, REDUCER)); assertThat(monitor.vertexStatusMap.get(MAPPER), is(sameInstance(succeeded))); assertThat(monitor.vertexStatusMap.get(REDUCER), is(sameInstance(running))); assertThat(monitor.progressCountsMap.keySet(), hasItems(MAPPER, REDUCER)); TezProgressMonitor.VertexProgress expectedMapperState = new TezProgressMonitor.VertexProgress(2, 1, 3, 4, 5, DAGStatus.State.RUNNING); assertThat(monitor.progressCountsMap.get(MAPPER), is(equalTo(expectedMapperState))); TezProgressMonitor.VertexProgress expectedReducerState = new TezProgressMonitor.VertexProgress(3, 2, 1, 0, 1, DAGStatus.State.RUNNING); assertThat(monitor.progressCountsMap.get(REDUCER), is(equalTo(expectedReducerState))); }
@Override public VertexStatus getVertexStatus(String vertexName, Set<StatusGetOpts> statusOptions) throws IOException, TezException { return realClient.getVertexStatus(vertexName, statusOptions); }
@Override public VertexStatus getVertexStatus( String vertexName, Set<StatusGetOpts> statusOptions ) throws IOException, TezException { return dagClient.getVertexStatus( vertexName, statusOptions ); }
@Override public VertexStatus getVertexStatus( String vertexName, Set<StatusGetOpts> statusOptions ) throws IOException, TezException { return dagClient.getVertexStatus( vertexName, statusOptions ); }
public VertexStatus getVertexStatus(String vertexName) { VertexStatus vs = null; try { vs = dagClient.getVertexStatus(vertexName, statusGetOpts); } catch (Exception e) { // Don't fail the job even if vertex status couldn't // be retrieved. log.warn("Cannot retrieve status for vertex " + vertexName, e); } return vs; }
VertexStatus vertexStatus = dagClient.getVertexStatus( flowNodeStats.getID(), null ); // no counters
VertexStatus vertexStatus = dagClient.getVertexStatus( flowNodeStats.getID(), null ); // no counters
private VertexStatus updateProgress( DAGClient dagClient, Set<StatusGetOpts> statusGetOpts ) { VertexStatus vertexStatus = null; try { vertexStatus = dagClient.getVertexStatus( getID(), statusGetOpts ); } catch( IOException | TezException exception ) { logWarn( "unable to get vertex status for: {}", getID(), exception ); } if( vertexStatus == null ) return null; Progress progress = vertexStatus.getProgress(); totalTaskCount = progress.getTotalTaskCount(); runningTaskCount = progress.getRunningTaskCount(); succeededTaskCount = progress.getSucceededTaskCount(); failedTaskCount = progress.getFailedTaskCount(); killedTaskCount = progress.getKilledTaskCount(); return vertexStatus; } }
private VertexStatus updateProgress( DAGClient dagClient, Set<StatusGetOpts> statusGetOpts ) { VertexStatus vertexStatus = null; try { vertexStatus = dagClient.getVertexStatus( getID(), statusGetOpts ); } catch( IOException | TezException exception ) { logWarn( "unable to get vertex status for: {}", getID(), exception ); } if( vertexStatus == null ) return null; Progress progress = vertexStatus.getProgress(); totalTaskCount = progress.getTotalTaskCount(); runningTaskCount = progress.getRunningTaskCount(); succeededTaskCount = progress.getSucceededTaskCount(); failedTaskCount = progress.getFailedTaskCount(); killedTaskCount = progress.getKilledTaskCount(); return vertexStatus; } }