public void setRuleDuration( Rule rule, long begin, long end ) { Map<String, Long> durations = ruleDurations.get( rule.getRulePhase() ); if( durations == null ) { durations = new LinkedHashMap<>(); ruleDurations.put( rule.getRulePhase(), durations ); } if( durations.containsKey( rule.getRuleName() ) ) throw new IllegalStateException( "duplicate rule found: " + rule.getRuleName() ); long duration = end - begin; // print these as we go if( duration > THRESHOLD_SECONDS * 1000 ) LOG.info( "rule: {}, took longer than {} seconds: {}", rule.getRuleName(), THRESHOLD_SECONDS, formatDurationFromMillis( duration ) ); durations.put( rule.getRuleName(), duration ); }
protected void finalizeNodeSliceCapture() { long startOfFinalPolling = System.currentTimeMillis(); long lastLog = 0; long retries = 0; boolean allNodesFinished; while( true ) { allNodesFinished = updateNodesStatus(); flowStepStats.recordChildStats(); if( allNodesFinished && flowStepStats.hasCapturedFinalDetail() ) break; if( ( System.currentTimeMillis() - startOfFinalPolling ) >= blockForCompletedChildDetailDuration ) break; if( System.currentTimeMillis() - lastLog > 1000 ) { if( !allNodesFinished ) flowStep.logInfo( "did not capture all completed node details, will retry in {}, prior retries: {}", formatDurationFromMillis( pollingInterval ), retries ); else flowStep.logInfo( "did not capture all completed slice details, will retry in {}, prior retries: {}", formatDurationFromMillis( pollingInterval ), retries ); lastLog = System.currentTimeMillis(); } retries++; sleepForPollingInterval(); } if( !allNodesFinished ) flowStep.logWarn( "unable to capture all completed node details or determine final state within configured duration: {}, configure property to increase wait duration: '{}'", formatDurationFromMillis( blockForCompletedChildDetailDuration ), CascadingStats.STATS_COMPLETE_CHILD_DETAILS_BLOCK_DURATION ); if( !flowStepStats.hasCapturedFinalDetail() ) flowStep.logWarn( "unable to capture all completed slice details within configured duration: {}, configure property to increase wait duration: '{}'", formatDurationFromMillis( blockForCompletedChildDetailDuration ), CascadingStats.STATS_COMPLETE_CHILD_DETAILS_BLOCK_DURATION ); }
private boolean updateAllTasks( TimelineClient timelineClient ) { if( allChildrenFinished ) return true; long startTime = System.currentTimeMillis(); int count = 0; for( FlowSliceStats sliceStats : sliceStatsMap.values() ) { if( sliceStats.getStatus().isFinished() ) continue; TaskStatus taskStatus = getTaskStatusFor( timelineClient, sliceStats.getProcessSliceID() ); updateSliceWith( (TezSliceStats) sliceStats, taskStatus, System.currentTimeMillis() ); count++; } if( count == 0 ) allChildrenFinished = true; logInfo( "updated {} slices in: {}", count, formatDurationFromMillis( System.currentTimeMillis() - startTime ) ); return sliceStatsMap.size() == getTotalTaskCount(); }
private boolean updateAllTasks( TimelineClient timelineClient ) { if( allChildrenFinished ) return true; long startTime = System.currentTimeMillis(); int count = 0; for( FlowSliceStats sliceStats : sliceStatsMap.values() ) { if( sliceStats.getStatus().isFinished() ) continue; TaskStatus taskStatus = getTaskStatusFor( timelineClient, sliceStats.getProcessSliceID() ); updateSliceWith( (TezSliceStats) sliceStats, taskStatus, System.currentTimeMillis() ); count++; } if( count == 0 ) allChildrenFinished = true; logInfo( "updated {} slices in: {}", count, formatDurationFromMillis( System.currentTimeMillis() - startTime ) ); return sliceStatsMap.size() == getTotalTaskCount(); }
private void notifyInterrupted() { if( interrupted.isEmpty() ) return; for( RuleResult ruleResult : interrupted ) getFlowLogger().logInfo( "rule registry: {}, planned longer than default duration, was cancelled", ruleResult.getRegistry().getName() ); if( interrupted.size() == registrySet.size() ) throw new PlannerException( "planner registry timeout exceeded for all registries: " + formatDurationFromMillis( registrySet.getPlannerTimeoutSec() * 1000 ) ); if( !registrySet.isIgnoreFailed() || success.isEmpty() ) rethrow( interrupted.get( 0 ).getPlannerException() ); }
getFlowLogger().logWarn( "planner cancelling long running registries past timeout period: {}, see RuleRegistrySet#setPlannerTimeoutSec() to change timeout", formatDurationFromMillis( registrySet.getPlannerTimeoutSec() * 1000 ) ); else getFlowLogger().logInfo( "first registry completed, planner cancelling remaining running registries: {}, successful: {}", futures.size(), success.size() );
LOG.warn( "no counters fetched, max num consecutive retries reached: {}, type: {}, status: {}", maxFetchAttempts, stats.getType(), stats.getStatus() ); else LOG.warn( "stale counters being returned, max num consecutive retries reached, age: {}, type: {}, status: {}", formatDurationFromMillis( currentTimeMillis() - lastFetch ), stats.getType(), stats.getStatus() );
String duration = formatDurationFromMillis( System.currentTimeMillis() - lastFetch );
int added = total - startSize; int remaining = getTotalTaskCount() - total; String duration = formatDurationFromMillis( System.currentTimeMillis() - startTime );
String duration = formatDurationFromMillis( System.currentTimeMillis() - lastFetch );
protected void planPhases( PlannerContext plannerContext, boolean logAsInfo, RuleResult ruleResult ) { ProcessLogger logger = plannerContext.getLogger(); for( PlanPhase phase : PlanPhase.values() ) // iterate in order, all planner phases { long beginPhase = System.currentTimeMillis(); logPhase( logger, logAsInfo, "starting rule phase: {}", phase ); try { switch( phase.getAction() ) { case Resolve: resolveElements( ruleResult ); break; case Rule: executeRulePhase( phase, plannerContext, ruleResult ); break; } } finally { long endPhase = System.currentTimeMillis(); ruleResult.setPhaseDuration( phase, beginPhase, endPhase ); logPhase( logger, logAsInfo, "ending rule phase: {}, duration: {}", phase, formatDurationFromMillis( endPhase - beginPhase ) ); } } }
public RuleResult exec( PlannerContext plannerContext, FlowElementGraph flowElementGraph ) { RuleResult ruleResult = new RuleResult( registry, flowElementGraph ); ProcessLogger logger = plannerContext.getLogger(); int size = flowElementGraph.vertexSet().size(); boolean logAsInfo = size >= ELEMENT_THRESHOLD; if( logAsInfo ) logger.logInfo( "elements in graph: {}, info logging threshold: {}, logging planner execution status", size, ELEMENT_THRESHOLD ); long beginExec = System.currentTimeMillis(); try { planPhases( plannerContext, logAsInfo, ruleResult ); } catch( Exception exception ) { ruleResult.setPlannerException( exception ); } finally { long endExec = System.currentTimeMillis(); ruleResult.setDuration( beginExec, endExec ); RuleResult.ResultStatus status = ruleResult.getResultStatus(); String duration = formatDurationFromMillis( endExec - beginExec ); logPhase( logger, logAsInfo, "rule registry completed: {}, with status: {}, and duration: {}", registry.getName(), status, duration ); } return ruleResult; }
int added = total - startSize; int remaining = getTotalTaskCount() - total; String duration = formatDurationFromMillis( System.currentTimeMillis() - startTime );
logInfo( " completed in: " + formatDurationFromMillis( flowStats.getDuration() ) ); else logInfo( " completed in: " + formatDurationFromMillis( flowStats.getDuration() ) + ", using cpu time: " + formatDurationFromMillis( totalSliceCPUSeconds ) );
protected RuleResult execPlannerFor( RuleRegistry ruleRegistry ) { flowPlanner.configRuleRegistryDefaults( ruleRegistry ); String registryName = ruleRegistry.getName(); RuleExec ruleExec = new RuleExec( traceWriter, ruleRegistry ); PlannerContext plannerContext = new PlannerContext( ruleRegistry, flowPlanner, flowDef, flow, traceWriter.isTransformTraceEnabled() ); RuleResult ruleResult = ruleExec.exec( plannerContext, flowElementGraph ); getFlowLogger().logInfo( "executed rule registry: {}, completed as: {}, in: {}", registryName, ruleResult.getResultStatus(), formatDurationFromMillis( ruleResult.getDuration() ) ); traceWriter.writeTracePlan( registryName, "completed-flow-element-graph", ruleResult.getAssemblyGraph() ); traceWriter.writeStats( plannerContext, ruleResult ); Exception plannerException; if( ruleResult.isSuccess() ) plannerException = flowPlanner.verifyResult( ruleResult ); else plannerException = ruleResult.getPlannerException(); // will be re-thrown below if( plannerException != null && plannerException instanceof PlannerException && ( (PlannerException) plannerException ).getElementGraph() != null ) traceWriter.writeTracePlan( registryName, "failed-source-element-graph", ( (PlannerException) plannerException ).getElementGraph() ); if( ruleResult.isSuccess() && plannerException != null ) rethrow( plannerException ); return ruleResult; }