public int getOrdinal() { return flowNode.getOrdinal(); }
private void addRemoteDebug( FlowNode flowNode, Vertex vertex ) { String value = System.getProperty( "test.debug.node", null ); if( Util.isEmpty( value ) ) return; if( !flowNode.getSourceElementNames().contains( value ) && asInt( value ) != flowNode.getOrdinal() ) return; LOG.warn( "remote debugging enabled with property: {}, on node: {}, with node id: {}", "test.debug.node", value, flowNode.getID() ); String opts = vertex.getTaskLaunchCmdOpts(); if( opts == null ) opts = ""; String address = System.getProperty( "test.debug.address", "localhost:5005" ).trim(); opts += " -agentlib:jdwp=transport=dt_socket,server=n,address=" + address + ",suspend=y"; vertex.setTaskLaunchCmdOpts( opts ); }
private void addRemoteDebug( FlowNode flowNode, Vertex vertex ) { String value = System.getProperty( "test.debug.node", null ); if( Util.isEmpty( value ) ) return; if( !flowNode.getSourceElementNames().contains( value ) && asInt( value ) != flowNode.getOrdinal() ) return; LOG.warn( "remote debugging enabled with property: {}, on node: {}, with node id: {}", "test.debug.node", value, flowNode.getID() ); String opts = vertex.getTaskLaunchCmdOpts(); if( opts == null ) opts = ""; String address = System.getProperty( "test.debug.address", "localhost:5005" ).trim(); opts += " -agentlib:jdwp=transport=dt_socket,server=n,address=" + address + ",suspend=y"; vertex.setTaskLaunchCmdOpts( opts ); }
private void addRemoteProfiling( FlowNode flowNode, Vertex vertex ) { String value = System.getProperty( "test.profile.node", null ); if( Util.isEmpty( value ) ) return; if( !flowNode.getSourceElementNames().contains( value ) && asInt( value ) != flowNode.getOrdinal() ) return; LOG.warn( "remote profiling enabled with property: {}, on node: {}, with node id: {}", "test.profile.node", value, flowNode.getID() ); String opts = vertex.getTaskLaunchCmdOpts(); if( opts == null ) opts = ""; String path = System.getProperty( "test.profile.path", "/tmp/jfr/" ); if( !path.endsWith( "/" ) ) path += "/"; LOG.warn( "remote profiling property: {}, logging to path: {}", "test.profile.path", path ); opts += String.format( " -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath=%1$s%2$s,disk=true,repository=%1$s%2$s", path, flowNode.getID() ); vertex.setTaskLaunchCmdOpts( opts ); }
private void addRemoteProfiling( FlowNode flowNode, Vertex vertex ) { String value = System.getProperty( "test.profile.node", null ); if( Util.isEmpty( value ) ) return; if( !flowNode.getSourceElementNames().contains( value ) && asInt( value ) != flowNode.getOrdinal() ) return; LOG.warn( "remote profiling enabled with property: {}, on node: {}, with node id: {}", "test.profile.node", value, flowNode.getID() ); String opts = vertex.getTaskLaunchCmdOpts(); if( opts == null ) opts = ""; String path = System.getProperty( "test.profile.path", "/tmp/jfr/" ); if( !path.endsWith( "/" ) ) path += "/"; LOG.warn( "remote profiling property: {}, logging to path: {}", "test.profile.path", path ); opts += String.format( " -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath=%1$s%2$s,disk=true,repository=%1$s%2$s", path, flowNode.getID() ); vertex.setTaskLaunchCmdOpts( opts ); }
private Configuration getNodeConfig(FlowNode node) { Configuration nodeConfig = HadoopUtil.copyConfiguration(this.getConfig()); ConfigurationSetter configSetter = new ConfigurationSetter(nodeConfig); this.initConfFromNodeConfigDef(node.getElementGraph(), configSetter); this.initConfFromStepConfigDef(configSetter); nodeConfig.set("cascading.flow.node.num", Integer.toString(node.getOrdinal())); return nodeConfig; }
int nodeOrdinal = flowNode.getOrdinal(); String nodeGraphName = String.format( "%s/%04d-%04d-step-node-sub-graph-%s.dot", rootPath, stepOrdinal, nodeOrdinal, canonicalHash( nodeGraph ) );
LOG.info( "flow node id: {}, ordinal: {}", flowNode.getID(), flowNode.getOrdinal() );
LOG.info( "flow node id: {}, ordinal: {}", flowNode.getID(), flowNode.getOrdinal() );
LOG.info( "flow node id: {}, ordinal: {}", flowNode.getID(), flowNode.getOrdinal() );
LOG.info( "flow node id: {}, ordinal: {}", flowNode.getID(), flowNode.getOrdinal() );
conf.set( "cascading.flow.node.num", Integer.toString( flowNode.getOrdinal() ) );
conf.set( "cascading.flow.node.num", Integer.toString( flowNode.getOrdinal() ) );
throw (CascadingException) throwable; throw new FlowException( "internal error during processor execution on node: " + flowNode.getOrdinal(), throwable );
throw (CascadingException) throwable; throw new FlowException( "internal error during processor execution on node: " + flowNode.getOrdinal(), throwable );
@Override public void initialize() throws Exception { configuration = new TezConfiguration( TezUtils.createConfFromUserPayload( getContext().getUserPayload() ) ); TezUtil.setMRProperties( getContext(), configuration, true ); try { HadoopUtil.initLog4j( configuration ); LOG.info( "cascading version: {}", configuration.get( "cascading.version", "" ) ); currentProcess = new Hadoop2TezFlowProcess( new FlowSession(), getContext(), configuration ); flowNode = deserializeBase64( configuration.getRaw( FlowNode.CASCADING_FLOW_NODE ), configuration, BaseFlowNode.class ); LOG.info( "flow node id: {}, ordinal: {}", flowNode.getID(), flowNode.getOrdinal() ); logMemory( LOG, "flow node id: " + flowNode.getID() + ", mem on start" ); } catch( Throwable throwable ) { if( throwable instanceof CascadingException ) throw (CascadingException) throwable; throw new FlowException( "internal error during processor configuration", throwable ); } }
@Override public void initialize() throws Exception { configuration = new TezConfiguration( TezUtils.createConfFromUserPayload( getContext().getUserPayload() ) ); TezUtil.setMRProperties( getContext(), configuration, true ); try { HadoopUtil.initLog4j( configuration ); LOG.info( "cascading version: {}", configuration.get( "cascading.version", "" ) ); currentProcess = new Hadoop2TezFlowProcess( new FlowSession(), getContext(), configuration ); flowNode = deserializeBase64( configuration.getRaw( FlowNode.CASCADING_FLOW_NODE ), configuration, BaseFlowNode.class ); LOG.info( "flow node id: {}, ordinal: {}", flowNode.getID(), flowNode.getOrdinal() ); logMemory( LOG, "flow node id: " + flowNode.getID() + ", mem on start" ); } catch( Throwable throwable ) { if( throwable instanceof CascadingException ) throw (CascadingException) throwable; throw new FlowException( "internal error during processor configuration", throwable ); } }