public void writeTransformPlan( String registryName, PlanPhase phase, Rule rule, int[] ordinals, GraphResult graphResult ) { if( isTransformTraceDisabled() ) return; String ruleName = String.format( "%02d-%s-%04d", phase.ordinal(), phase, addRule( rule ) ); for( int i = 1; i < ordinals.length; i++ ) ruleName = String.format( "%s-%04d", ruleName, ordinals[ i ] ); ruleName = String.format( "%s-%s", ruleName, graphResult.getRuleName() ); Path path = getFullTransformTracePath( registryName ).resolve( ruleName ); graphResult.writeDOTs( path.toString() ); markResult( graphResult, path ); }
protected Path getPlanTransformTracePath() { return applyScope( getStringProperty( System.getProperties(), properties, FlowPlanner.TRACE_PLAN_TRANSFORM_PATH ) ); }
public boolean isTransformTraceDisabled() { return getPlanTransformTracePath() == null; }
public void writeTransformPlan( String registryName, FlowElementGraph flowElementGraph, String name ) { if( isTransformTraceDisabled() ) return; if( flowElementGraph == null ) { processLogger.logInfo( "cannot write phase assembly trace, flowElementGraph is null" ); return; } Path file = getFullTransformTracePath( registryName ).resolve( name ).normalize(); processLogger.logInfo( "writing phase assembly trace: {}, to: {}", name, file ); flowElementGraph.writeDOT( file.toString() ); }
private void writeTransformTrace( RuleResult ruleResult, PlanPhase phase, Rule rule, ElementGraph parent, ElementGraph child, GraphResult result ) { if( traceWriter.isTransformTraceDisabled() ) return; int[] path = child != null ? ruleResult.getPathFor( parent, child ) : ruleResult.getPathFor( parent ); traceWriter.writeTransformPlan( registry.getName(), phase, rule, path, result ); } }
TraceWriter traceWriter = new TraceWriter( flow ); RuleSetExec ruleSetExec = new RuleSetExec( traceWriter, this, flow, ruleRegistrySet, flowDef, flowElementGraph ); traceWriter.writeTracePlan( null, "0-initial-flow-element-graph", flowElementGraph ); traceWriter.writeFinal( "1-final-flow-registry", ruleResult ); traceWriter.writeTracePlan( null, "2-final-flow-element-graph", finalFlowElementGraph ); traceWriter.writeTracePlan( null, "3-final-flow-step-graph", flowStepGraph ); traceWriter.writeTracePlanSteps( "4-final-flow-steps", flowStepGraph );
@Test public void testRuleEngine() { RuleRegistry ruleRegistry = new RuleRegistry(); ruleRegistry.addElementFactory( NonTapFactory.TEMP_TAP, new NonTapFactory() ); PlannerContext plannerContext = new PlannerContext( ruleRegistry, null, null, null, true ); ruleRegistry.addRule( new RuleAssert( PlanPhase.PreResolveAssembly, new LoneGroupExpression(), "lone group assertion" ) ); ruleRegistry.addRule( new RuleInsertionTransformer( PlanPhase.PreResolveAssembly, new TestGroupGroupExpression(), IntermediateTapElementFactory.TEMP_TAP ) ); try { new RuleExec( new TraceWriter(), ruleRegistry ).executeRulePhase( PlanPhase.PreResolveAssembly, plannerContext, new RuleResult( new LoneGroupAssertionGraph() ) ); fail(); } catch( PlannerException exception ) { // do nothing } new RuleExec( new TraceWriter(), ruleRegistry ).executeRulePhase( PlanPhase.PreResolveAssembly, plannerContext, new RuleResult( new HashJoinSameSourceGraph() ) ); }
protected RuleResult execPlannerFor( RuleRegistry ruleRegistry ) { flowPlanner.configRuleRegistryDefaults( ruleRegistry ); String registryName = ruleRegistry.getName(); RuleExec ruleExec = new RuleExec( traceWriter, ruleRegistry ); PlannerContext plannerContext = new PlannerContext( ruleRegistry, flowPlanner, flowDef, flow, traceWriter.isTransformTraceEnabled() ); RuleResult ruleResult = ruleExec.exec( plannerContext, flowElementGraph ); getFlowLogger().logInfo( "executed rule registry: {}, completed as: {}, in: {}", registryName, ruleResult.getResultStatus(), formatDurationFromMillis( ruleResult.getDuration() ) ); traceWriter.writeTracePlan( registryName, "completed-flow-element-graph", ruleResult.getAssemblyGraph() ); traceWriter.writeStats( plannerContext, ruleResult ); Exception plannerException; if( ruleResult.isSuccess() ) plannerException = flowPlanner.verifyResult( ruleResult ); else plannerException = ruleResult.getPlannerException(); // will be re-thrown below if( plannerException != null && plannerException instanceof PlannerException && ( (PlannerException) plannerException ).getElementGraph() != null ) traceWriter.writeTracePlan( registryName, "failed-source-element-graph", ( (PlannerException) plannerException ).getElementGraph() ); if( ruleResult.isSuccess() && plannerException != null ) rethrow( plannerException ); return ruleResult; }
public void writeStats( PlannerContext plannerContext, RuleResult ruleResult ) { Path path = getPlanStatsPath(); if( path == null ) return; File file = path.resolve( String.format( "planner-stats-%s-%s.txt", ruleResult.getRegistry().getName(), ruleResult.getResultStatus() ) ).toFile(); processLogger.logInfo( "writing planner stats to: {}", file ); file.getParentFile().mkdirs(); try( PrintWriter writer = new PrintWriter( file ) ) { Flow flow = plannerContext.getFlow(); Map<Object, Object> configAsProperties = flow.getConfigAsProperties(); writer.format( "cascading version: %s, build: %s\n", emptyOrValue( Version.getReleaseFull() ), emptyOrValue( Version.getReleaseBuild() ) ); writer.format( "application id: %s\n", emptyOrValue( AppProps.getApplicationID( configAsProperties ) ) ); writer.format( "application name: %s\n", emptyOrValue( AppProps.getApplicationName( configAsProperties ) ) ); writer.format( "application version: %s\n", emptyOrValue( AppProps.getApplicationVersion( configAsProperties ) ) ); writer.format( "platform: %s\n", emptyOrValue( flow.getPlatformInfo() ) ); writer.format( "frameworks: %s\n", emptyOrValue( AppProps.getApplicationFrameworks( configAsProperties ) ) ); writer.println(); ruleResult.writeStats( writer ); } catch( IOException exception ) { processLogger.logError( "could not write stats", exception ); } }
public void writeTracePlan( String registryName, String fileName, FlowStepGraph stepGraph ) { Path path = getPlanTracePath(); if( path == null ) return; if( stepGraph == null ) { processLogger.logInfo( "cannot write step plan, stepGraph is null" ); return; } if( registryName != null ) path = path.resolve( registryName ); Path filePath = path.resolve( String.format( "%s.dot", fileName ) ); File file = filePath.toFile(); processLogger.logInfo( "writing trace step plan: {}", file ); stepGraph.writeDOT( file.toString() ); }
public boolean isTransformTraceEnabled() { return !isTransformTraceDisabled(); }
int addRule( Rule rule ) { return addRule( rule.getRulePhase().getLevel(), rule ); }
@Test public void testSubGraphIterator() { RuleRegistry ruleRegistry = new RuleRegistry(); ruleRegistry.addElementFactory( NonTapFactory.TEMP_TAP, new NonTapFactory() ); PlannerContext plannerContext = new PlannerContext( ruleRegistry, null, null, null, true ); ruleRegistry.addRule( new RuleInsertionTransformer( PlanPhase.PreResolveAssembly, new TestCheckpointExpression(), IntermediateTapElementFactory.TEMP_TAP ) ); // ruleRegistry.addRule( new RuleContractedTransform( PlanPhase.PreResolve, new NoOpPipeExpression() ) ); RuleResult ruleResult = new RuleExec( new TraceWriter(), ruleRegistry ).executeRulePhase( PlanPhase.PreResolveAssembly, plannerContext, new RuleResult( new StandardElementGraph() ) ); FlowElementGraph flowElementGraph = ruleResult.getAssemblyGraph(); SubGraphIterator iterator = new ExpressionSubGraphIterator( new PlannerContext(), new NoGroupTapExpressionGraph(), new TapGroupTapExpressionGraph(), flowElementGraph ); while( iterator.hasNext() ) assertNotNull( iterator.next() ); }
public void writeTransformPlan( String registryName, List<? extends ElementGraph> flowElementGraphs, PlanPhase phase, String subName ) { if( isTransformTraceDisabled() ) return; if( flowElementGraphs == null || flowElementGraphs.isEmpty() ) { processLogger.logInfo( "cannot write phase step trace, flowElementGraphs is empty" ); return; } for( int i = 0; i < flowElementGraphs.size(); i++ ) { ElementGraph flowElementGraph = flowElementGraphs.get( i ); String name = String.format( "%02d-%s-%s-%04d.dot", phase.ordinal(), phase, subName, i ); Path file = getFullTransformTracePath( registryName ).resolve( name ).normalize(); processLogger.logInfo( "writing phase step trace: {}, to: {}", name, file ); flowElementGraph.writeDOT( file.toString() ); } }
public void writeFinal( String fileName, RuleResult ruleResult ) { Path path = getPlanTracePath(); if( path == null ) return; Path filePath = path.resolve( String.format( "%s-%s.txt", fileName, ruleResult.getRegistry().getName() ) ); File file = filePath.toFile(); processLogger.logInfo( "writing final registry: {}", file ); try( PrintWriter writer = new PrintWriter( file ) ) { writer.println( "filename names winning rule registry" ); } catch( IOException exception ) { processLogger.logError( "could not write final registry", exception ); } }
private void runSubGraphIteratorRotate( FlowElementGraph elementGraph, int numSubGraphs ) { RuleRegistry ruleRegistry = new RuleRegistry(); PlannerContext plannerContext = new PlannerContext( ruleRegistry ); ruleRegistry.addRule( new RemoveNoOpPipeTransformer() ); RuleResult ruleResult = new RuleExec( new TraceWriter(), ruleRegistry ).executeRulePhase( PlanPhase.PreResolveAssembly, plannerContext, new RuleResult( elementGraph ) ); FlowElementGraph flowElementGraph = ruleResult.getAssemblyGraph(); flowElementGraph.writeDOT( getPlanPath() + "/node.dot" ); ExpressionSubGraphIterator iterator = new ExpressionSubGraphIterator( new PlannerContext(), new TestNoGroupTapExpressionGraph(), new TestConsecutiveTapsExpressionGraph(), false, flowElementGraph ); iterator.getContractedGraph().writeDOT( getPlanPath() + "/node-contracted.dot" ); int count = 0; while( iterator.hasNext() && count < 10 ) { ElementGraph next = iterator.next(); assertNotNull( next ); next.writeDOT( getPlanPath() + "/pipeline/" + count + "-graph.dot" ); count++; } assertEquals( "wrong number of sub-graphs", numSubGraphs, count ); }
public void writeTransformPlan( String registryName, Map<ElementGraph, List<? extends ElementGraph>> subGraphsMap, PlanPhase phase, String subName ) { if( isTransformTraceDisabled() ) return; if( subGraphsMap == null || subGraphsMap.isEmpty() ) { processLogger.logInfo( "cannot write phase node trace, subGraphs is empty" ); return; } int stepCount = 0; for( Map.Entry<ElementGraph, List<? extends ElementGraph>> entry : subGraphsMap.entrySet() ) { List<? extends ElementGraph> flowElementGraphs = entry.getValue(); for( int i = 0; i < flowElementGraphs.size(); i++ ) { ElementGraph flowElementGraph = flowElementGraphs.get( i ); String name = String.format( "%02d-%s-%s-%04d-%04d.dot", phase.ordinal(), phase, subName, stepCount, i ); Path file = getFullTransformTracePath( registryName ).resolve( name ); processLogger.logInfo( "writing phase node trace: {}, to: {}", name, file ); flowElementGraph.writeDOT( file.toString() ); } stepCount++; } }
public void writeTracePlan( String registryName, String fileName, ElementGraph elementGraph ) { Path path = getPlanTracePath(); if( path == null ) return; if( elementGraph == null ) { processLogger.logInfo( "cannot write trace element plan, elementGraph is null" ); return; } if( registryName != null ) path = path.resolve( registryName ); Path filePath = path.resolve( String.format( "%s-%s.dot", fileName, canonicalHash( elementGraph ) ) ); File file = filePath.toFile(); processLogger.logInfo( "writing trace element plan: {}", file ); String filename = file.toString(); elementGraph.writeDOT( filename ); }
protected Path getPlanStatsPath() { return applyScope( getStringProperty( System.getProperties(), properties, FlowPlanner.TRACE_STATS_PATH ) ); }
protected Path getFullTransformTracePath( String registryName ) { Path planTransformTracePath = getPlanTransformTracePath(); if( planTransformTracePath == null ) return null; return planTransformTracePath.resolve( registryName ); }