@SuppressWarnings ( "unchecked" ) private List<StepMetaDataCombi> getSubSteps( Transformation transformation, StepMetaDataCombi combi ) { HashMap<String, Transformation> config = ( (Optional<HashMap<String, Transformation>>) transformation .getConfig( TransMetaConverter.SUB_TRANSFORMATIONS_KEY ) ) .orElse( Maps.newHashMap() ); StepMetaInterface smi = combi.stepMeta.getStepMetaInterface(); return config.keySet().stream() .filter( key -> stepHasDependency( combi, smi, key ) ) .flatMap( key -> opsToSteps( config.get( key ) ).stream() ) .collect( Collectors.toList() ); }
@Test public void simpleConvert() { TransMeta meta = new TransMeta(); meta.setFilename( "fileName" ); meta.addStep( new StepMeta( "stepName", stepMetaInterface ) ); Transformation trans = TransMetaConverter.convert( meta ); assertThat( trans.getId(), is( meta.getFilename() ) ); assertThat( trans.getOperations().size(), is( 1 ) ); assertThat( trans.getOperations().get( 0 ).getId(), is( "stepName" ) ); }
@Test public void transWithHops() { TransMeta meta = new TransMeta(); meta.setFilename( "fileName" ); StepMeta from = new StepMeta( "step1", stepMetaInterface ); meta.addStep( from ); StepMeta to = new StepMeta( "step2", stepMetaInterface ); meta.addStep( to ); meta.addTransHop( new TransHopMeta( from, to ) ); Transformation trans = TransMetaConverter.convert( meta ); assertThat( trans.getId(), is( meta.getFilename() ) ); assertThat( trans.getOperations().size(), is( 2 ) ); assertThat( trans.getHops().size(), is( 1 ) ); assertThat( trans.getHops().get( 0 ).getFrom().getId(), is( from.getName() ) ); assertThat( trans.getHops().get( 0 ).getTo().getId(), is( to.getName() ) ); assertThat( trans.getHops().stream().map( Hop::getType ).collect( Collectors.toList() ), everyItem( is( Hop.TYPE_NORMAL ) ) ); }
@Test public void testRemovingDisabledInputSteps() { TransMeta trans = new TransMeta(); StepMeta inputToBeRemoved = new StepMeta( "InputToBeRemoved", stepMetaInterface ); trans.addStep( inputToBeRemoved ); StepMeta inputToStay = new StepMeta( "InputToStay", stepMetaInterface ); trans.addStep( inputToStay ); StepMeta inputReceiver1 = new StepMeta( "InputReceiver1", stepMetaInterface ); trans.addStep( inputReceiver1 ); StepMeta inputReceiver2 = new StepMeta( "InputReceiver2", stepMetaInterface ); trans.addStep( inputReceiver2 ); TransHopMeta hop1 = new TransHopMeta( inputToBeRemoved, inputReceiver1, false ); TransHopMeta hop2 = new TransHopMeta( inputToStay, inputReceiver1 ); TransHopMeta hop3 = new TransHopMeta( inputToBeRemoved, inputReceiver2, false ); trans.addTransHop( hop1 ); trans.addTransHop( hop2 ); trans.addTransHop( hop3 ); Transformation transformation = TransMetaConverter.convert( trans ); List<String> steps = transformation.getOperations().stream().map( op -> op.getId() ).collect( Collectors.toList() ); assertThat( "Only 2 ops should exist", steps.size(), is( 2 ) ); assertThat( steps, hasItems( "InputToStay", "InputReceiver1" ) ); List<String> hops = transformation.getHops().stream().map( hop -> hop.getId() ).collect( Collectors.toList() ); assertThat( "Only 1 hop should exist", hops.size(), is( 1 ) ); assertThat( hops, hasItems( "InputToStay -> InputReceiver1" ) ); }
private List<StepMetaDataCombi> opsToSteps( Transformation transformation ) { Map<Operation, StepMetaDataCombi> operationToCombi = transformation.getOperations().stream() .collect( toMap( Function.identity(), op -> { StepMetaDataCombi combi = new StepMetaDataCombi(); combi.stepMeta = StepMeta.fromXml( (String) op.getConfig().get( TransMetaConverter.STEP_META_CONF_KEY ) ); try { combi.data = new StepDataInterfaceWebSocketEngineAdapter( op, messageEventService ); List<StepMetaDataCombi> subSteps = getSubSteps( transformation, combi ); combi.step = new StepInterfaceWebSocketEngineAdapter( op, messageEventService, combi.stepMeta, transMeta, combi.data, this, subSteps ); } catch ( KettleException e ) { //TODO: treat the exception e.printStackTrace(); } combi.meta = combi.stepMeta.getStepMetaInterface(); combi.stepname = combi.stepMeta.getName(); return combi; } ) ); return new ArrayList<>( operationToCombi.values() ); }
@Test public void transIdFromRepo() throws Exception { TransMeta meta = new TransMeta(); meta.setName( "transName" ); Transformation trans = TransMetaConverter.convert( meta ); assertThat( trans.getId(), is( "/transName" ) ); }
@Test public void errorHops() throws Exception { TransMeta meta = new TransMeta(); meta.setFilename( "fileName" ); StepMeta from = new StepMeta( "step1", stepMetaInterface ); meta.addStep( from ); StepMeta to = new StepMeta( "step2", stepMetaInterface ); meta.addStep( to ); meta.addTransHop( new TransHopMeta( from, to ) ); StepMeta error = new StepMeta( "errorHandler", stepMetaInterface ); meta.addStep( error ); TransHopMeta errorHop = new TransHopMeta( from, error ); errorHop.setErrorHop( true ); meta.addTransHop( errorHop ); Transformation trans = TransMetaConverter.convert( meta ); Map<String, List<Hop>> hops = trans.getHops().stream().collect( Collectors.groupingBy( Hop::getType ) ); List<Hop> normalHops = hops.get( Hop.TYPE_NORMAL ); assertThat( normalHops.size(), is( 1 ) ); assertThat( normalHops.get( 0 ).getTo().getId(), is( "step2" ) ); List<Hop> errorHops = hops.get( Hop.TYPE_ERROR ); assertThat( errorHops.size(), is( 1 ) ); assertThat( errorHops.get( 0 ).getTo().getId(), is( "errorHandler" ) ); assertThat( hops.values().stream() .flatMap( List::stream ) .map( Hop::getFrom ).map( Operation::getId ) .collect( Collectors.toList() ), everyItem( equalTo( "step1" ) ) ); }
@Test public void testMultipleDisabledHops() { TransMeta trans = new TransMeta(); StepMeta input = new StepMeta( "Input", stepMetaInterface ); trans.addStep( input ); StepMeta step1 = new StepMeta( "Step1", stepMetaInterface ); trans.addStep( step1 ); StepMeta step2 = new StepMeta( "Step2", stepMetaInterface ); trans.addStep( step2 ); StepMeta step3 = new StepMeta( "Step3", stepMetaInterface ); trans.addStep( step3 ); TransHopMeta hop1 = new TransHopMeta( input, step1, false ); TransHopMeta hop2 = new TransHopMeta( step1, step2, false ); TransHopMeta hop3 = new TransHopMeta( step2, step3, false ); trans.addTransHop( hop1 ); trans.addTransHop( hop2 ); trans.addTransHop( hop3 ); Transformation transformation = TransMetaConverter.convert( trans ); assertThat( "Trans has steps though all of them should be removed", transformation.getOperations().size(), is( 0 ) ); assertThat( "Trans has hops though all of them should be removed", transformation.getHops().size(), is( 0 ) ); }
private void subscribeToOpLogging() throws KettleException { transformation.getOperations().stream().forEach( operation -> { try { messageEventService.addHandler( Util.getOperationLogEvent( operation.getId() ), new MessageEventHandler() { @Override public void execute( Message message ) throws MessageEventHandlerExecutionException { PDIEvent<RemoteSource, LogEntry> event = (PDIEvent<RemoteSource, LogEntry>) message; LogEntry logEntry = event.getData(); StepInterface stepInterface = findStepInterface( operation.getId(), 0 ); if ( stepInterface != null ) { LogChannelInterface logChannel = stepInterface.getLogChannel(); logToChannel( logChannel, logEntry ); } else { // Could not find step, log at transformation level instead logToChannel( getLogChannel(), logEntry ); } } @Override public String getIdentifier() { return OPERATION_LOG + operation.getId(); } } ); } catch ( HandlerRegistrationException e ) { getLogChannel().logError( "Error registering message handlers", e ); } } ); }
transformation.getOperations().stream().map( op -> op.getId() ).collect( Collectors.toList() ); assertThat( "Only 5 ops should exist", steps.size(), is( 5 ) ); assertThat( steps, hasItems( "Start", "WithEnabledHop", "ShouldStay", "WithEnabledAndDisabledHops", "AfterEnabledDisabled" ) ); List<String> hops = transformation.getHops().stream().map( hop -> hop.getId() ).collect( Collectors.toList() ); assertThat( "Only 4 hops should exist", hops.size(), is( 4 ) ); assertThat( hops, hasItems( "Start -> WithEnabledHop", "WithEnabledHop -> ShouldStay",
@Test public void transConfigItemsNoNameSpecified() throws Exception { TransMeta meta = new TransMeta(); Transformation trans = TransMetaConverter.convert( meta ); assertThat( trans.getConfig().get( TransMetaConverter.TRANS_META_NAME_CONF_KEY ), is( TransMetaConverter.TRANS_DEFAULT_NAME ) ); assertThat( (String) trans.getConfig().get( TransMetaConverter.TRANS_META_CONF_KEY ), startsWith( "<transformation>" ) ); }
@Test public void transConfigItems() throws Exception { TransMeta meta = new TransMeta(); meta.setName( "foo" ); Transformation trans = TransMetaConverter.convert( meta ); assertThat( trans.getConfig().get( TransMetaConverter.TRANS_META_NAME_CONF_KEY ), is( "foo" ) ); assertThat( (String) trans.getConfig().get( TransMetaConverter.TRANS_META_CONF_KEY ), startsWith( "<transformation>" ) ); }
@Test public void testIncludesSubTransformations() throws Exception { TransMeta parentTransMeta = new TransMeta( getClass().getResource( "trans-meta-converter-parent.ktr" ).getPath() ); Transformation transformation = TransMetaConverter.convert( parentTransMeta ); @SuppressWarnings( { "unchecked", "ConstantConditions" } ) HashMap<String, Transformation> config = (HashMap<String, Transformation>) transformation.getConfig( TransMetaConverter.SUB_TRANSFORMATIONS_KEY ).get(); assertEquals( 1, config.size() ); assertNotNull( config.get( "file://" + getClass().getResource( "trans-meta-converter-sub.ktr" ).getPath() ) ); }
@Test public void testIncludesSubTransformationsFromRepository() throws Exception { TransMeta parentTransMeta = new TransMeta( getClass().getResource( "trans-meta-converter-parent.ktr" ).getPath() ); Repository repository = mock( Repository.class ); TransMeta transMeta = new TransMeta( ); RepositoryDirectoryInterface repositoryDirectory = new RepositoryDirectory( null, "public"); String directory = getClass().getResource( "" ).toString().replace( File.separator, "/" ); when( repository.findDirectory( "public" ) ).thenReturn( repositoryDirectory ); when( repository.loadTransformation( "trans-meta-converter-sub.ktr", repositoryDirectory, null, true, null ) ).thenReturn( transMeta ); parentTransMeta.setRepository( repository ); parentTransMeta.setRepositoryDirectory( repositoryDirectory ); parentTransMeta.setVariable( "Internal.Entry.Current.Directory", "public" ); Transformation transformation = TransMetaConverter.convert( parentTransMeta ); @SuppressWarnings( { "unchecked", "ConstantConditions" } ) HashMap<String, Transformation> config = (HashMap<String, Transformation>) transformation.getConfig( TransMetaConverter.SUB_TRANSFORMATIONS_KEY ).get(); assertEquals( 1, config.size() ); assertNotNull( config.get( "public/trans-meta-converter-sub.ktr" ) ); }
@Test public void lazyConversionTurnedOff() throws KettleException { KettleEnvironment.init(); TransMeta transMeta = new TransMeta(); CsvInputMeta csvInputMeta = new CsvInputMeta(); csvInputMeta.setLazyConversionActive( true ); StepMeta csvInput = new StepMeta( "Csv", csvInputMeta ); transMeta.addStep( csvInput ); TableInputMeta tableInputMeta = new TableInputMeta(); tableInputMeta.setLazyConversionActive( true ); StepMeta tableInput = new StepMeta( "Table", tableInputMeta ); transMeta.addStep( tableInput ); Transformation trans = TransMetaConverter.convert( transMeta ); TransMeta cloneMeta; String transMetaXml = (String) trans.getConfig().get( TransMetaConverter.TRANS_META_CONF_KEY ); Document doc; try { doc = XMLHandler.loadXMLString( transMetaXml ); Node stepNode = XMLHandler.getSubNode( doc, "transformation" ); cloneMeta = new TransMeta( stepNode, null ); } catch ( KettleXMLException | KettleMissingPluginsException e ) { throw new RuntimeException( e ); } assertThat( ( (CsvInputMeta) cloneMeta.findStep( "Csv" ).getStepMetaInterface() ).isLazyConversionActive(), is( false ) ); assertThat( ( (TableInputMeta) cloneMeta.findStep( "Table" ).getStepMetaInterface() ).isLazyConversionActive(), is( false ) ); }