/** * Executes the transformation. This method will prepare the transformation for execution and then start all the * threads associated with the transformation and its steps. * * @param arguments the arguments * @throws KettleException if the transformation could not be prepared (initialized) */ public void execute( String[] arguments ) throws KettleException { prepareExecution( arguments ); startThreads(); }
protected void executeTrans( Trans trans ) throws KettleException { trans.prepareExecution( null ); trans.startThreads(); trans.waitUntilFinished(); }
public Optional<Result> execute( List<RowMetaAndData> rows ) throws KettleException { if ( rows.isEmpty() || stopped ) { return Optional.empty(); } Trans subtrans = this.createSubtrans(); running.add( subtrans ); parentTrans.addActiveSubTransformation( subTransName, subtrans ); // Pass parameter values passParametersToTrans( subtrans, rows.get( 0 ) ); Result result = new Result(); result.setRows( rows ); subtrans.setPreviousResult( result ); subtrans.prepareExecution( this.parentTrans.getArguments() ); List<RowMetaAndData> rowMetaAndData = new ArrayList<>(); subtrans.getSteps().stream() .filter( c -> c.step.getStepname().equalsIgnoreCase( subStep ) ) .findFirst() .ifPresent( c -> c.step.addRowListener( new RowAdapter() { @Override public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) { rowMetaAndData.add( new RowMetaAndData( rowMeta, row ) ); } } ) ); subtrans.startThreads(); subtrans.waitUntilFinished(); updateStatuses( subtrans ); running.remove( subtrans ); Result subtransResult = subtrans.getResult(); subtransResult.setRows( rowMetaAndData ); return Optional.of( subtransResult ); }
@Before public void setUp() throws Exception { executor = StepMockUtil.getStep( TransExecutor.class, TransExecutorMeta.class, "TransExecutorUnitTest" ); executor = spy( executor ); TransMeta internalTransMeta = mock( TransMeta.class ); doReturn( internalTransMeta ).when( executor ).loadExecutorTransMeta(); internalTrans = spy( new Trans() ); internalTrans.setLog( mock( LogChannelInterface.class ) ); doNothing().when( internalTrans ).prepareExecution( any( String[].class ) ); doNothing().when( internalTrans ).startThreads(); doNothing().when( internalTrans ).waitUntilFinished(); doNothing().when( executor ).discardLogLines( any( TransExecutorData.class ) ); doReturn( internalTrans ).when( executor ).createInternalTrans(); internalResult = new Result(); doReturn( internalResult ).when( internalTrans ).getResult(); meta = new TransExecutorMeta(); data = new TransExecutorData(); }
/** * PDI-14948 - Execution of trans with no steps never ends */ @Test ( timeout = 1000 ) public void transWithNoStepsIsNotEndless() throws Exception { Trans transWithNoSteps = new Trans( new TransMeta() ); transWithNoSteps = spy( transWithNoSteps ); transWithNoSteps.prepareExecution( new String[] {} ); transWithNoSteps.startThreads(); // check trans lifecycle is not corrupted verify( transWithNoSteps ).fireTransStartedListeners(); verify( transWithNoSteps ).fireTransFinishedListeners(); }
@Before public void beforeTest() throws KettleException { meta = new TransMeta(); trans = new Trans( meta ); trans.setLog( Mockito.mock( LogChannelInterface.class ) ); trans.prepareExecution( null ); trans.startThreads(); }
@Override public void run() { try { trans.prepareExecution( args ); // Do we capture data? // if ( transPreviewDelegate.isActive() ) { transPreviewDelegate.capturePreviewData( trans, transMeta.getSteps() ); } initialized = true; } catch ( KettleException e ) { log.logError( trans.getName() + ": preparing transformation execution failed", e ); checkErrorVisuals(); } halted = trans.hasHaltedSteps(); if ( trans.isReadyToStart() ) { checkStartThreads(); // After init, launch the threads. } else { initialized = false; running = false; checkErrorVisuals(); } } };
@Test public void doesNotWriteRowOnTimeWhenStopped() throws KettleException, InterruptedException { TransMeta transMeta = new TransMeta( getClass().getResource( "safe-stop.ktr" ).getPath() ); Trans trans = new Trans( transMeta ); trans.prepareExecution( new String[] {} ); trans.getSteps().get( 1 ).step.addRowListener( new RowAdapter() { @Override public void rowWrittenEvent( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException { trans.safeStop(); } } ); trans.startThreads(); trans.waitUntilFinished(); assertEquals( 1, trans.getSteps().get( 0 ).step.getLinesWritten() ); assertEquals( 1, trans.getSteps().get( 1 ).step.getLinesRead() ); } }
@GET @Path( "/prepare/{id : .+}" ) @Produces( { MediaType.APPLICATION_JSON } ) public TransformationStatus prepareTransformation( @PathParam( "id" ) String id ) { Trans trans = CarteResource.getTransformation( id ); try { CarteObjectEntry entry = CarteResource.getCarteObjectEntry( id ); TransConfiguration transConfiguration = CarteSingleton.getInstance().getTransformationMap().getConfiguration( entry ); TransExecutionConfiguration executionConfiguration = transConfiguration.getTransExecutionConfiguration(); // Set the appropriate logging, variables, arguments, replay date, ... // etc. trans.setArguments( executionConfiguration.getArgumentStrings() ); trans.setReplayDate( executionConfiguration.getReplayDate() ); trans.setSafeModeEnabled( executionConfiguration.isSafeModeEnabled() ); trans.setGatheringMetrics( executionConfiguration.isGatheringMetrics() ); trans.injectVariables( executionConfiguration.getVariables() ); trans.setPreviousResult( executionConfiguration.getPreviousResult() ); trans.prepareExecution( null ); } catch ( KettleException e ) { e.printStackTrace(); } return getTransformationStatus( id ); }
@Before public void setUp() throws KettleException { KettleLogStore.setLogChannelInterfaceFactory( logChannelFactory ); when( logChannelFactory.create( any(), any() ) ).thenReturn( logChannel ); StepMeta stepMeta = new StepMeta( "BaseStreamStep", meta ); TransMeta transMeta = new TransMeta(); transMeta.addStep( stepMeta ); Trans trans = new Trans( transMeta ); baseStreamStep = new BaseStreamStep( stepMeta, stepData, 1, transMeta, trans ); baseStreamStep.source = streamSource; baseStreamStep.window = streamWindow; baseStreamStep.setParentVariableSpace( new Variables() ); StepMetaDataCombi stepMetaDataCombi = new StepMetaDataCombi(); stepMetaDataCombi.step = baseStreamStep; stepMetaDataCombi.data = stepData; stepMetaDataCombi.stepMeta = stepMeta; stepMetaDataCombi.meta = meta; trans.prepareExecution( new String[ 0 ] ); trans.getSteps().add( stepMetaDataCombi ); }
/** * This checks transformation initialization when using many copies to one next step * * @throws KettleException */ @Test public void testManyToOneCopies() throws KettleException { prepareStepMetas_x2_1(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 2 rowsets finally", 2, rowsets.size() ); assertEquals( "We have 4 steps: 2 copies of producer and 2 copies of consumer", 3, trans.getSteps().size() ); // Ok, examine initialized steps now. StepInterface stepOne0 = getStepByName( S10 ); assertTrue( "1 step have no input row sets", stepOne0.getInputRowSets().isEmpty() ); assertEquals( "1 step have 1 output rowsets", 1, stepOne0.getOutputRowSets().size() ); StepInterface stepOne1 = getStepByName( S11 ); assertTrue( "1 step have no input row sets", stepOne1.getInputRowSets().isEmpty() ); assertEquals( "1 step have 1 output rowsets", 1, stepOne1.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( S20 ); Assert.assertEquals( "2.0 step have 2 input row sets", 2, stepTwo0.getInputRowSets().size() ); Assert.assertTrue( "2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty() ); }
/** * This checks transformation initialization when using one to many copies * * @throws KettleException */ @Test public void testOneToManyCopies() throws KettleException { prepareStepMetas_1_x2(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 2 rowsets finally", 2, rowsets.size() ); assertEquals( "We have 3 steps: one producer and 2 copies of consumer", 3, trans.getSteps().size() ); // Ok, examine initialized steps now. StepInterface stepOne = getStepByName( S10 ); assertTrue( "1 step have no input row sets", stepOne.getInputRowSets().isEmpty() ); assertEquals( "1 step have 2 output rowsets", 2, stepOne.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( S20 ); Assert.assertEquals( "2.0 step have 12 input row sets", 1, stepTwo0.getInputRowSets().size() ); Assert.assertTrue( "2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty() ); StepInterface stepTwo1 = getStepByName( S21 ); Assert.assertEquals( "2.1 step have 1 input row sets", 1, stepTwo1.getInputRowSets().size() ); Assert.assertTrue( "2.1 step have no output row sets", stepTwo1.getOutputRowSets().isEmpty() ); }
/** * Test one to one partitioning step transformation organization. * * @throws KettleException */ @Test public void testOneToPartitioningSchema() throws KettleException { prepareStepMetas_1_cl1(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 2 rowsets finally", 2, rowsets.size() ); assertEquals( "We have 3 steps: 1 producer and 2 copies of consumer since it is partitioned", 3, trans.getSteps() .size() ); // Ok, examine initialized steps now. StepInterface stepOne0 = getStepByName( S10 ); assertTrue( "1 step have no input row sets", stepOne0.getInputRowSets().isEmpty() ); assertEquals( "1 step have 2 output rowsets", 2, stepOne0.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( SP20 ); assertEquals( "2.0 step have one input row sets", 1, stepTwo0.getInputRowSets().size() ); assertTrue( "2.0 step have no output rowsets", stepTwo0.getOutputRowSets().isEmpty() ); StepInterface stepTwo1 = getStepByName( SP21 ); Assert.assertEquals( "2.1 step have 1 input row sets", 1, stepTwo1.getInputRowSets().size() ); Assert.assertTrue( "2.1 step have no output row sets", stepTwo1.getOutputRowSets().isEmpty() ); }
/** * Test 'Swim lines partitioning' * * @throws KettleException */ @Test public void testSwimLanesPartitioning() throws KettleException { prepareStepMetas_cl1_cl1(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 2 rowsets finally", 2, rowsets.size() ); assertEquals( "We have 3 steps: 1 producer and 2 copies of consumer since it is partitioned", 4, trans.getSteps() .size() ); // Ok, examine initialized steps now. StepInterface stepOne0 = getStepByName( SP10 ); assertTrue( "1.0 step have no input row sets", stepOne0.getInputRowSets().isEmpty() ); assertEquals( "1.0 step have 1 output rowsets", 1, stepOne0.getOutputRowSets().size() ); StepInterface stepOne1 = getStepByName( SP11 ); assertTrue( "1.1 step have no input row sets", stepOne1.getInputRowSets().isEmpty() ); assertEquals( "1.1 step have 1 output rowsets", 1, stepOne1.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( SP20 ); assertEquals( "2.0 step have 2 input row sets", 1, stepTwo0.getInputRowSets().size() ); assertTrue( "2.0 step have no output rowsets", stepTwo0.getOutputRowSets().isEmpty() ); StepInterface stepTwo2 = getStepByName( SP21 ); assertTrue( "2.2 step have no output row sets", stepTwo2.getOutputRowSets().isEmpty() ); assertEquals( "2.2 step have 2 output rowsets", 1, stepTwo2.getInputRowSets().size() ); }
/** * This checks transformation initialization when using many to many copies. * * @throws KettleException */ @Test public void testManyToManyCopies() throws KettleException { prepareStepMetas_x2_x2(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 2 rowsets finally", 2, rowsets.size() ); assertEquals( "We have 4 steps: 2 copies of producer and 2 copies of consumer", 4, trans.getSteps().size() ); // Ok, examine initialized steps now. StepInterface stepOne0 = getStepByName( S10 ); assertTrue( "1 step have no input row sets", stepOne0.getInputRowSets().isEmpty() ); assertEquals( "1 step have 1 output rowsets", 1, stepOne0.getOutputRowSets().size() ); StepInterface stepOne1 = getStepByName( S11 ); assertTrue( "1 step have no input row sets", stepOne1.getInputRowSets().isEmpty() ); assertEquals( "1 step have 1 output rowsets", 1, stepOne1.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( S20 ); Assert.assertEquals( "2.0 step have 1 input row sets", 1, stepTwo0.getInputRowSets().size() ); Assert.assertTrue( "2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty() ); StepInterface stepTwo1 = getStepByName( S21 ); Assert.assertEquals( "2.1 step have 1 input row sets", 1, stepTwo1.getInputRowSets().size() ); Assert.assertTrue( "2.1 step have no output row sets", stepTwo1.getOutputRowSets().isEmpty() ); }
/** * This is a case when step running in many copies meets partitioning one. * * @throws KettleException */ @Test public void testManyCopiesToPartitioningFlow() throws KettleException { prepareStepMetas_x2_cl1(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 4 rowsets finally since repartitioning happens", 4, rowsets.size() ); assertEquals( "We have 4 steps: 2 producer copies and 2 copies of consumer since consumer is partitioned", 4, trans .getSteps().size() ); // Ok, examine initialized steps now. StepInterface stepOne0 = getStepByName( S10 ); assertTrue( "1.0 step have no input row sets", stepOne0.getInputRowSets().isEmpty() ); assertEquals( "1.0 step have 2 output rowsets", 2, stepOne0.getOutputRowSets().size() ); StepInterface stepOne1 = getStepByName( S11 ); assertTrue( "1.1 step have no input row sets", stepOne1.getInputRowSets().isEmpty() ); assertEquals( "1.1 step have 2 output rowsets", 2, stepOne1.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( SP20 ); assertTrue( "2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty() ); assertEquals( "2.0 step have 2 input rowsets", 2, stepTwo0.getInputRowSets().size() ); StepInterface stepTwo2 = getStepByName( SP21 ); assertTrue( "2.1 step have no output row sets", stepTwo2.getOutputRowSets().isEmpty() ); assertEquals( "2.2 step have 2 input rowsets", 2, stepTwo2.getInputRowSets().size() ); }
private JoinRows getJoinRows() throws Exception { StepMeta stepMeta = new StepMeta(); TransMeta transMeta = new TransMeta(); Trans trans = new Trans( transMeta ); transMeta.clear(); transMeta.addStep( stepMeta ); transMeta.setStep( 0, stepMeta ); stepMeta.setName( "test" ); trans.setLog( mock( LogChannelInterface.class ) ); trans.prepareExecution( null ); trans.startThreads(); return new JoinRows( stepMeta, null, 0, transMeta, trans ); }
/** * This is PDI-12140 case. 2 steps with same partitions ID's count but different partitioner. This is not a swim lines * cases and we need repartitioning here. * * @throws KettleException */ @Test public void testDifferentPartitioningFlow() throws KettleException { prepareStepMetas_cl1_cl2(); trans.prepareExecution( new String[] {} ); List<RowSet> rowsets = trans.getRowsets(); assertTrue( !rowsets.isEmpty() ); assertEquals( "We have 4 rowsets finally since repartitioning happens", 4, rowsets.size() ); assertEquals( "We have 4 steps: 2 producer copies and 2 copies of consumer since they both partitioned", 4, trans .getSteps().size() ); // Ok, examine initialized steps now. StepInterface stepOne0 = getStepByName( SP10 ); assertTrue( "1.0 step have no input row sets", stepOne0.getInputRowSets().isEmpty() ); assertEquals( "1.0 step have 2 output rowsets", 2, stepOne0.getOutputRowSets().size() ); StepInterface stepOne1 = getStepByName( SP11 ); assertTrue( "1.1 step have no input row sets", stepOne1.getInputRowSets().isEmpty() ); assertEquals( "1.1 step have 2 output rowsets", 2, stepOne1.getOutputRowSets().size() ); StepInterface stepTwo0 = getStepByName( SP20 ); assertTrue( "2.0 step have no output row sets", stepTwo0.getOutputRowSets().isEmpty() ); assertEquals( "2.0 step have 1 input rowsets", 2, stepTwo0.getInputRowSets().size() ); StepInterface stepTwo2 = getStepByName( SP21 ); assertTrue( "2.1 step have no output row sets", stepTwo2.getOutputRowSets().isEmpty() ); assertEquals( "2.2 step have 2 input rowsets", 2, stepTwo2.getInputRowSets().size() ); }
executorTrans.prepareExecution( getTrans().getArguments() );
@Test( expected = KettleException.class ) public void failsWhenGivenNonSingleThreadSteps() throws Exception { Meta metaInterface = createMeta(); PluginRegistry plugReg = PluginRegistry.getInstance(); String id = plugReg.getPluginId( StepPluginType.class, metaInterface ); assertNotNull( "pluginId", id ); StepMeta stepMeta = new StepMeta( id, "stepMetrics", metaInterface ); stepMeta.setDraw( true ); TransMeta transMeta = new TransMeta(); transMeta.setName( "failsWhenGivenNonSingleThreadSteps" ); transMeta.addStep( stepMeta ); Trans trans = new Trans( transMeta ); trans.prepareExecution( null ); SingleThreadedTransExecutor executor = new SingleThreadedTransExecutor( trans ); executor.init(); } }