/** * Calculate the lineage for the specified step only... * * @param stepMeta * The step to calculate the lineage for. * @throws KettleStepException * In case there is an exception calculating the lineage. This is usually caused by unavailable data sources * etc. */ private void calculateLineage( StepMeta stepMeta ) throws KettleStepException { RowMetaInterface outputMeta = transMeta.getStepFields( stepMeta ); // The lineage is basically a calculation of origin for each output of a certain step. // for ( ValueMetaInterface valueMeta : outputMeta.getValueMetaList() ) { StepMeta originStepMeta = transMeta.findStep( valueMeta.getOrigin(), stepMeta ); if ( originStepMeta != null ) { /* List<StepMeta> list = */fieldStepsMap.get( originStepMeta ); } } }
/** * Create a new field mapping between source and target steps. * * @param shell the shell of the parent window * @param sourceFields the source fields * @param targetFields the target fields * @param fieldMapping the list of source to target mappings to default to (can be empty but not null) * @throws KettleException in case something goes wrong during the field mapping */ public static final void generateFieldMapping( Shell shell, RowMetaInterface sourceFields, RowMetaInterface targetFields, List<SourceToTargetMapping> fieldMapping ) throws KettleException { // Build the mapping: let the user decide!! String[] source = sourceFields.getFieldNames(); for ( int i = 0; i < source.length; i++ ) { ValueMetaInterface v = sourceFields.getValueMeta( i ); source[ i ] += EnterMappingDialog.STRING_ORIGIN_SEPARATOR + v.getOrigin() + ")"; } String[] target = targetFields.getFieldNames(); EnterMappingDialog dialog = new EnterMappingDialog( shell, source, target, fieldMapping ); List<SourceToTargetMapping> newMapping = dialog.open(); if ( newMapping != null ) { fieldMapping.clear(); fieldMapping.addAll( newMapping ); } }
public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) throws KettleStepException { if ( prev != null ) { // Lookup: we do a lookup on the natural keys for ( int i = 0; i < keyLookup.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( keyStream[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_DELETE, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), tableName, keyLookup[i], keyStream[i], v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } } }
public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) throws KettleStepException { if ( prev != null ) { /* DEBUG CHECK THIS */ // Insert dateMask fields : read/write for ( int i = 0; i < fieldTable.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( fieldStream[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ_WRITE, transMeta.getName(), stepMeta.getName(), databaseMeta.getDatabaseName(), transMeta.environmentSubstitute( tableName ), fieldTable[i], fieldStream[i], v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } } }
public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) throws KettleStepException { if ( prev != null ) { /* DEBUG CHECK THIS */ // Insert dateMask fields : read/write for ( int i = 0; i < fieldTable.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( fieldStream[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ_WRITE, transMeta.getName(), stepMeta.getName(), databaseMeta.getDatabaseName(), transMeta.environmentSubstitute( tableName ), fieldTable[i], fieldStream[i], v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } } }
@Override public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) throws KettleStepException { if ( prev != null ) { /* DEBUG CHECK THIS */ // Insert dateMask fields : read/write for ( int i = 0; i < fieldTable.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( fieldStream[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ_WRITE, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), transMeta.environmentSubstitute( tableName ), fieldTable[i], fieldStream[i], v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } } }
public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) throws KettleStepException { if ( prev != null ) { /* DEBUG CHECK THIS */ // Insert dateMask fields : read/write for ( int i = 0; i < fieldTable.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( fieldStream[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ_WRITE, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), transMeta.environmentSubstitute( tableName ), fieldTable[i], fieldStream[i], v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } } }
public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) throws KettleStepException { if ( prev != null ) { /* DEBUG CHECK THIS */ // Insert dateMask fields : read/write for ( int i = 0; i < fieldTable.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( fieldStream[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ_WRITE, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), transMeta.environmentSubstitute( tableName ), fieldTable[i], fieldStream[i], v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } } }
public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) throws KettleStepException { if ( prev != null ) { /* DEBUG CHECK THIS */ // Insert dateMask fields : read/write for ( int i = 0; i < fieldTable.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( fieldStream[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ_WRITE, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), transMeta.environmentSubstitute( tableName ), fieldTable[i], fieldStream[i], v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } } }
public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) throws KettleStepException { if ( prev != null ) { // Lookup: we do a lookup on the natural keys for ( int i = 0; i < keyLookup.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( keyStream[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), tableName, keyLookup[i], keyStream[i], v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } // Insert update fields : read/write for ( int i = 0; i < updateLookup.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( updateStream[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ_WRITE, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), tableName, updateLookup[i], updateStream[i], v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } } }
@Override public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) throws KettleStepException { if ( prev != null ) { // Lookup: we do a lookup on the natural keys for ( int i = 0; i < keyLookup.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( keyStream[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), tableName, keyLookup[i], keyStream[i], v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } // Update fields : read/write for ( int i = 0; i < updateLookup.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( updateStream[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_UPDATE, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), tableName, updateLookup[i], updateStream[i], v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } } }
public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) throws KettleStepException { if ( prev != null ) { // Lookup: we do a lookup on the natural keys for ( int i = 0; i < keyLookup.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( keyStream[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), tableName, keyLookup[i], keyStream[i], v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } // Insert update fields : read/write for ( int i = 0; i < updateLookup.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( updateStream[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ_WRITE, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), tableName, updateLookup[i], updateStream[i], v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } } }
@Test public void testPrepareExecutionResultsFields() throws Exception { TransExecutorMeta meta = new TransExecutorMeta(); meta = spy( meta ); RowMetaInterface row = mock( RowMetaInterface.class ); StepMeta nextStep = mock( StepMeta.class ); meta.setExecutionResultTargetStepMeta( nextStep ); meta.setExecutionTimeField( "time" ); StepMeta parent = mock( StepMeta.class ); doReturn( parent ).when( meta ).getParentStepMeta(); when( parent.getName() ).thenReturn( "parent step" ); meta.prepareExecutionResultsFields( row, nextStep ); // make sure we get the name of the parent step meta... used for the origin step verify( parent ).getName(); ArgumentCaptor<ValueMetaInterface> argumentCaptor = ArgumentCaptor.forClass( ValueMetaInterface.class ); verify( row ).addValueMeta( argumentCaptor.capture() ); assertEquals( "parent step", argumentCaptor.getValue().getOrigin() ); }
@Test public void testPrepareExecutionResultsFileFields() throws Exception { TransExecutorMeta meta = new TransExecutorMeta(); meta = spy( meta ); RowMetaInterface row = mock( RowMetaInterface.class ); StepMeta nextStep = mock( StepMeta.class ); meta.setResultFilesTargetStepMeta( nextStep ); meta.setResultFilesFileNameField( "file_name" ); StepMeta parent = mock( StepMeta.class ); doReturn( parent ).when( meta ).getParentStepMeta(); when( parent.getName() ).thenReturn( "parent step" ); meta.prepareExecutionResultsFileFields( row, nextStep ); // make sure we get the name of the parent step meta... used for the origin step verify( parent ).getName(); ArgumentCaptor<ValueMetaInterface> argumentCaptor = ArgumentCaptor.forClass( ValueMetaInterface.class ); verify( row ).addValueMeta( argumentCaptor.capture() ); assertEquals( "parent step", argumentCaptor.getValue().getOrigin() ); }
@Test public void testPrepareResultsRowsFields() throws Exception { TransExecutorMeta meta = new TransExecutorMeta(); String[] outputFieldNames = new String[]{ "one", "two" }; int[] outputFieldTypes = new int[]{ 0, 1 }; int[] outputFieldLength = new int[]{ 4, 8 }; int[] outputFieldPrecision = new int[]{ 2, 4 }; meta.setOutputRowsField( outputFieldNames ); meta.setOutputRowsType( outputFieldTypes ); meta.setOutputRowsLength( outputFieldLength ); meta.setOutputRowsPrecision( outputFieldPrecision ); meta = spy( meta ); RowMetaInterface row = mock( RowMetaInterface.class ); StepMeta parent = mock( StepMeta.class ); doReturn( parent ).when( meta ).getParentStepMeta(); when( parent.getName() ).thenReturn( "parent step" ); meta.prepareResultsRowsFields( row ); // make sure we get the name of the parent step meta... used for the origin step verify( parent, times( outputFieldNames.length ) ).getName(); ArgumentCaptor<ValueMetaInterface> argumentCaptor = ArgumentCaptor.forClass( ValueMetaInterface.class ); verify( row, times( outputFieldNames.length ) ).addValueMeta( argumentCaptor.capture() ); assertEquals( "parent step", argumentCaptor.getValue().getOrigin() ); }
@Override public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepinfo, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) { // The keys are read-only... for ( int i = 0; i < streamKeyField1.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( streamKeyField1[i] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ, transMeta.getName(), stepinfo.getName(), databaseMeta .getDatabaseName(), tablename, tableKeyField[i], streamKeyField1[i], v != null ? v.getOrigin() : "?", "", BaseMessages.getString( PKG, "DatabaseLookupMeta.Impact.Key" ) ); impact.add( ii ); } // The Return fields are read-only too... for ( int i = 0; i < returnValueField.length; i++ ) { DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ, transMeta.getName(), stepinfo.getName(), databaseMeta.getDatabaseName(), tablename, returnValueField[i], "", "", "", BaseMessages.getString( PKG, "DatabaseLookupMeta.Impact.ReturnValue" ) ); impact.add( ii ); } }
@Override public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) { // The keys are read-only... for ( int i = 0; i < keyField.length; i++ ) { ValueMetaInterface v = prev.searchValueMeta( keyField[ i ] ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ_WRITE, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), tablename, keyLookup[ i ], keyField[ i ], v != null ? v.getOrigin() : "?", "", useHash ? BaseMessages.getString( PKG, "CombinationLookupMeta.ReadAndInsert.Label" ) : BaseMessages .getString( PKG, "CombinationLookupMeta.LookupAndInsert.Label" ) ); impact.add( ii ); } // Do we lookup-on the hash-field? if ( useHash ) { DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_READ_WRITE, transMeta.getName(), stepMeta.getName(), databaseMeta.getDatabaseName(), tablename, hashField, "", "", "", BaseMessages.getString( PKG, "CombinationLookupMeta.KeyLookup.Label" ) ); impact.add( ii ); } }
public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) { if ( truncateTable ) { DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_TRUNCATE, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), tableName, "", "", "", "", "Truncate of table" ); impact.add( ii ); } // The values that are entering this step are in "prev": if ( prev != null ) { for ( int i = 0; i < prev.size(); i++ ) { ValueMetaInterface v = prev.getValueMeta( i ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_WRITE, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), tableName, v.getName(), v.getName(), v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } } }
public void analyseImpact( List<DatabaseImpact> impact, TransMeta transMeta, StepMeta stepMeta, RowMetaInterface prev, String[] input, String[] output, RowMetaInterface info, Repository repository, IMetaStore metaStore ) { if ( truncateTable ) { DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_TRUNCATE, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), tablename, "", "", "", "", "Truncate of table" ); impact.add( ii ); } // The values that are entering this step are in "prev": if ( prev != null ) { for ( int i = 0; i < prev.size(); i++ ) { ValueMetaInterface v = prev.getValueMeta( i ); DatabaseImpact ii = new DatabaseImpact( DatabaseImpact.TYPE_IMPACT_WRITE, transMeta.getName(), stepMeta.getName(), databaseMeta .getDatabaseName(), tablename, v.getName(), v.getName(), v != null ? v.getOrigin() : "?", "", "Type = " + v.toStringMeta() ); impact.add( ii ); } } }
@Test public void testGetFields() throws KettleStepException { CreditCardValidatorMeta meta = new CreditCardValidatorMeta(); meta.setDefault(); meta.setResultFieldName( "The Result Field" ); meta.setCardType( "The Card Type Field" ); meta.setNotValidMsg( "Is Card Valid" ); RowMeta rowMeta = new RowMeta(); meta.getFields( rowMeta, "this step", null, null, new Variables(), null, null ); assertEquals( 3, rowMeta.size() ); assertEquals( "The Result Field", rowMeta.getValueMeta( 0 ).getName() ); assertEquals( ValueMetaInterface.TYPE_BOOLEAN, rowMeta.getValueMeta( 0 ).getType() ); assertEquals( "this step", rowMeta.getValueMeta( 0 ).getOrigin() ); assertEquals( "The Card Type Field", rowMeta.getValueMeta( 1 ).getName() ); assertEquals( ValueMetaInterface.TYPE_STRING, rowMeta.getValueMeta( 1 ).getType() ); assertEquals( "this step", rowMeta.getValueMeta( 1 ).getOrigin() ); assertEquals( "Is Card Valid", rowMeta.getValueMeta( 2 ).getName() ); assertEquals( ValueMetaInterface.TYPE_STRING, rowMeta.getValueMeta( 2 ).getType() ); assertEquals( "this step", rowMeta.getValueMeta( 2 ).getOrigin() ); } }