private static boolean isValidRowMeta( RowMetaInterface rowMeta ) { for ( ValueMetaInterface value : rowMeta.getValueMetaList() ) { String name = value.getName(); if ( name == null || name.isEmpty() ) { return false; } } return true; }
private List<String> getOutputFieldNames( final TransMeta transMeta, final String stepName ) { final Map<String, RowMetaInterface> targetFieldsMap = getOutputRowMetaInterfaces( transMeta, transMeta.findStep( stepName ), null, false ); final List<String> targetFieldNames = new ArrayList(); if ( targetFieldsMap != null ) { final List<ValueMetaInterface> fieldValues = targetFieldsMap.values().iterator().next().getValueMetaList(); for ( final ValueMetaInterface fieldValue : fieldValues ) { targetFieldNames.add( fieldValue.getName() ); } } return targetFieldNames; }
/** * Calculate the lineage for the specified step only... * * @param stepMeta * The step to calculate the lineage for. * @throws KettleStepException * In case there is an exception calculating the lineage. This is usually caused by unavailable data sources * etc. */ private void calculateLineage( StepMeta stepMeta ) throws KettleStepException { RowMetaInterface outputMeta = transMeta.getStepFields( stepMeta ); // The lineage is basically a calculation of origin for each output of a certain step. // for ( ValueMetaInterface valueMeta : outputMeta.getValueMetaList() ) { StepMeta originStepMeta = transMeta.findStep( valueMeta.getOrigin(), stepMeta ); if ( originStepMeta != null ) { /* List<StepMeta> list = */fieldStepsMap.get( originStepMeta ); } } }
/** * Sort the entire vector, if it is not empty. */ void quickSort( List<Object[]> elements ) throws KettleException { if ( elements.size() > 0 ) { Collections.sort( elements, data.rowComparator ); long nrConversions = 0L; for ( ValueMetaInterface valueMeta : data.outputRowMeta.getValueMetaList() ) { nrConversions += valueMeta.getNumberOfBinaryStringConversions(); valueMeta.setNumberOfBinaryStringConversions( 0L ); } if ( log.isDetailed() ) { logDetailed( BaseMessages.getString( PKG, "SortRows.Detailed.ReportNumberOfBinaryStringConv", nrConversions ) ); } } }
public void getFields( RowMetaInterface row, String origin, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException { try { List<CheckResultInterface> remarks = new ArrayList<CheckResultInterface>(); RowMetaAndData rowMetaAndData = RowGenerator.buildRow( this, remarks, origin ); if ( !remarks.isEmpty() ) { StringBuilder stringRemarks = new StringBuilder(); for ( CheckResultInterface remark : remarks ) { stringRemarks.append( remark.toString() ).append( Const.CR ); } throw new KettleStepException( stringRemarks.toString() ); } for ( ValueMetaInterface valueMeta : rowMetaAndData.getRowMeta().getValueMetaList() ) { valueMeta.setOrigin( origin ); } row.mergeRowMeta( rowMetaAndData.getRowMeta() ); } catch ( Exception e ) { throw new KettleStepException( e ); } }
public void initializeColumns( RowMetaInterface inputRowMeta ) { if ( inputRowMeta == null ) { BaseMessages.getString( PKG, "RulesData.InitializeColumns.InputRowMetaIsNull" ); return; } // Create objects for insertion into the rules engine List<ValueMetaInterface> columns = inputRowMeta.getValueMetaList(); // This array must 1-1 match the row[] feteched by getRow() columnList = new Column[columns.size()]; for ( int i = 0; i < columns.size(); i++ ) { ValueMetaInterface column = columns.get( i ); Column c = new Column( true ); c.setName( column.getName() ); c.setType( column.getTypeDesc() ); c.setPayload( null ); columnList[i] = c; } }
public RowMetaInterface updateFields( RowMetaInterface prev, MonetDBBulkLoaderData data ) { // update the field table from the fields coming from the previous step RowMetaInterface tableFields = new RowMeta(); List<ValueMetaInterface> fields = prev.getValueMetaList(); fieldTable = new String[fields.size()]; fieldStream = new String[fields.size()]; fieldFormatOk = new boolean[fields.size()]; int idx = 0; for ( ValueMetaInterface field : fields ) { ValueMetaInterface tableField = field.clone(); tableFields.addValueMeta( tableField ); fieldTable[idx] = field.getName(); fieldStream[idx] = field.getName(); fieldFormatOk[idx] = true; idx++; } data.keynrs = new int[getFieldStream().length]; for ( int i = 0; i < data.keynrs.length; i++ ) { data.keynrs[i] = i; } return tableFields; }
protected Object[] writeToTable( RowMetaInterface rowMeta, Object[] r ) throws KettleException { // see if we need to truncate any string fields try { int index = 0; List<ValueMetaInterface> valueMetas = rowMeta.getValueMetaList(); for ( ValueMetaInterface valueMeta : valueMetas ) { Object valueData = r[index]; if ( valueData != null ) { if ( valueMeta.getType() == ValueMetaInterface.TYPE_STRING ) { String str = valueMeta.getString( valueData ); int len = valueMeta.getLength(); if ( len < 1 ) { len = MonetDBDatabaseMeta.DEFAULT_VARCHAR_LENGTH; } if ( str.length() > len ) { // TODO log this event str = str.substring( 0, len ); } r[index] = str; } } index++; } } catch ( Exception e ) { throw new KettleException( "Error serializing rows of data to the psql command", e ); } return super.writeToTable( rowMeta, r ); }
@Override protected Map<String, RowMetaInterface> getInputRowMetaInterfaces( GetXMLDataMeta meta ) { Map<String, RowMetaInterface> inputRows = getInputFields( meta ); if ( inputRows == null ) { inputRows = new HashMap<>(); } // Get some boolean flags from the meta for easier access boolean isInFields = meta.isInFields(); boolean isAFile = meta.getIsAFile(); boolean isAUrl = meta.isReadUrl(); // only add resource fields if we are NOT getting the xml or file from a field if ( !isInFields || isAFile || isAUrl ) { RowMetaInterface stepFields = getOutputFields( meta ); RowMetaInterface clone = stepFields.clone(); // if there are previous steps providing data, we should remove them from the set of "resource" fields for ( RowMetaInterface rowMetaInterface : inputRows.values() ) { for ( ValueMetaInterface valueMetaInterface : rowMetaInterface.getValueMetaList() ) { try { clone.removeValueMeta( valueMetaInterface.getName() ); } catch ( KettleValueException e ) { // could not find it in the output, skip it } } } inputRows.put( RESOURCE, clone ); } return inputRows; }
/** * putRow is used to copy a row, to the alternate rowset(s) This should get priority over everything else! * (synchronized) If distribute is true, a row is copied only once to the output rowsets, otherwise copies are sent to * each rowset! * * @param row The row to put to the destination rowset(s). * @throws KettleStepException */ @Override public void putRow( RowMetaInterface rowMeta, Object[] row ) throws KettleStepException { if ( rowMeta != null ) { if ( !allowEmptyFieldNamesAndTypes ) { // check row meta for empty field name (BACKLOG-18004) for ( ValueMetaInterface vmi : rowMeta.getValueMetaList() ) { if ( StringUtils.isBlank( vmi.getName() ) ) { throw new KettleStepException( "Please set a field name for all field(s) that have 'null'." ); } if ( vmi.getType() <= 0 ) { throw new KettleStepException( "Please set a value for the missing field(s) type." ); } } } } getRowHandler().putRow( rowMeta, row ); }
private void ok() { try { stringRowMeta = new RowMeta(); for ( ValueMetaInterface valueMeta : rowMeta.getValueMetaList() ) { ValueMetaInterface stringValueMeta = ValueMetaFactory.cloneValueMeta( valueMeta, ValueMetaInterface.TYPE_STRING ); stringRowMeta.addValueMeta( stringValueMeta ); } List<Object[]> list = new ArrayList<Object[]>(); // Now read all the rows in the dialog, including the empty rows... // for ( int i = 0; i < wFields.getItemCount(); i++ ) { TableItem item = wFields.getTable().getItem( i ); Object[] row = getRowForData( item, i + 1 ); list.add( row ); } outputList = list; dispose(); } catch ( Exception e ) { new ErrorDialog( shell, "Error", BaseMessages.getString( PKG, "EditRowsDialog.ErrorConvertingData" ), e ); } }
@Test public void getFieldWithValueUsedTwice() throws KettleStepException { databaseLookupMeta.setReturnValueField( new String[] { "match", "match", "mismatch" } ); databaseLookupMeta.setReturnValueNewName( new String[] { "v1", "v2", "v3" } ); ValueMetaInterface v1 = new ValueMetaString( "match" ); ValueMetaInterface v2 = new ValueMetaString( "match1" ); RowMetaInterface[] info = new RowMetaInterface[1]; info[0] = new RowMeta(); info[0].setValueMetaList( Arrays.asList( v1, v2 ) ); ValueMetaInterface r1 = new ValueMetaString( "value" ); RowMetaInterface row = new RowMeta(); row.setValueMetaList( new ArrayList<ValueMetaInterface>( Arrays.asList( r1 ) ) ); databaseLookupMeta.getFields( row, "", info, null, null, null, null ); List<ValueMetaInterface> expectedRow = Arrays.asList( new ValueMetaInterface[] { new ValueMetaString( "value" ), new ValueMetaString( "v1" ), new ValueMetaString( "v2" ), } ); assertEquals( 3, row.getValueMetaList().size() ); for ( int i = 0; i < 3; i++ ) { assertEquals( expectedRow.get( i ).getName(), row.getValueMetaList().get( i ).getName() ); } }
for ( ValueMetaInterface inputField : inputFields.getValueMetaList() ) { if ( row.searchValueMeta( inputField.getName() ) == null ) { row.addValueMeta( inputField );
public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException { meta = (FixedInputMeta) smi; data = (FixedInputData) sdi; if ( first ) { first = false; data.outputRowMeta = new RowMeta(); meta.getFields( data.outputRowMeta, getStepname(), null, null, this, repository, metaStore ); // The conversion logic for when the lazy conversion is turned of is simple: // Pretend it's a lazy conversion object anyway and get the native type during conversion. // data.convertRowMeta = data.outputRowMeta.clone(); for ( ValueMetaInterface valueMeta : data.convertRowMeta.getValueMetaList() ) { valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING ); } if ( meta.isHeaderPresent() ) { readOneRow( false ); // skip this row. } } Object[] outputRowData = readOneRow( true ); if ( outputRowData == null ) { // no more input to be expected... setOutputDone(); return false; } putRow( data.outputRowMeta, outputRowData ); // copy row to possible alternate rowset(s). if ( checkFeedback( getLinesInput() ) ) { logBasic( BaseMessages.getString( PKG, "FixedInput.Log.LineNumber", Long.toString( getLinesInput() ) ) ); } return true; }
@Test public void testGetFields_OutputValueRenames_WillRenameOutputIfValueMetaExist() throws KettleStepException { ValueMetaInterface valueMeta1 = new ValueMetaBase( "valueMeta1" ); ValueMetaInterface valueMeta2 = new ValueMetaBase( "valueMeta2" ); RowMetaInterface rowMeta = new RowMeta(); rowMeta.addValueMeta( valueMeta1 ); rowMeta.addValueMeta( valueMeta2 ); List<MappingValueRename> outputValueRenames = new ArrayList<MappingValueRename>(); outputValueRenames.add( new MappingValueRename( "valueMeta2", "valueMeta1" ) ); MappingOutputMeta meta = new MappingOutputMeta(); meta.setOutputValueRenames( outputValueRenames ); meta.getFields( rowMeta, null, info, nextStep, space, repository, metaStore ); //we must not add additional field assertEquals( 2, rowMeta.getValueMetaList().size() ); //we must not keep the first value meta since we want to rename second assertEquals( valueMeta1, rowMeta.getValueMeta( 0 ) ); //the second value meta must be other than we want to rename since we already have value meta with such name assertFalse( "valueMeta1".equals( rowMeta.getValueMeta( 1 ).getName() ) ); //the second value meta must be other than we want to rename since we already have value meta with such name. //It must be renamed according the rules from the #RowMeta assertTrue( "valueMeta1_1".equals( rowMeta.getValueMeta( 1 ).getName() ) ); }
for ( ValueMetaInterface valueMeta : row.getValueMetaList() ) { valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL );
List ls = rmi.getValueMetaList(); for ( Object l : ls ) { ValueMetaBase vmb = (ValueMetaBase) l;
for ( ValueMetaInterface valueMeta : data.rowMeta.getValueMetaList() ) { valueMeta.setOrigin( getStepname() );
@Test public void outputRowMetasAreNotSharedAmongSeveralStreams() throws Exception { RowSet rs1 = new SingleRowRowSet(); RowSet rs2 = new SingleRowRowSet(); when( mockHelper.trans.isRunning() ).thenReturn( true ); BaseStep baseStep = new BaseStep( mockHelper.stepMeta, mockHelper.stepDataInterface, 0, mockHelper.transMeta, mockHelper.trans ); baseStep.setStopped( false ); baseStep.setRepartitioning( StepPartitioningMeta.PARTITIONING_METHOD_NONE ); baseStep.setOutputRowSets( Arrays.asList( rs1, rs2 ) ); for ( RowSet rowSet : baseStep.getOutputRowSets() ) { assertNull( "RowMeta should be null, since no calls were done", rowSet.getRowMeta() ); } RowMetaInterface rowMeta = new RowMeta(); rowMeta.addValueMeta( new ValueMetaString( "string" ) ); rowMeta.addValueMeta( new ValueMetaInteger( "integer" ) ); baseStep.putRow( rowMeta, new Object[] { "a", 1 } ); RowMetaInterface meta1 = rs1.getRowMeta(); RowMetaInterface meta2 = rs2.getRowMeta(); assertNotNull( meta1 ); assertNotNull( meta2 ); // content is same for ( ValueMetaInterface meta : meta1.getValueMetaList() ) { assertTrue( meta.getName(), meta2.exists( meta ) ); } // whereas instances differ assertFalse( meta1 == meta2 ); }
@Test public void testgetResultRowMeta() { RowMetaInterface rm = StringSearchResult.getResultRowMeta(); assertNotNull( rm ); assertEquals( 4, rm.getValueMetaList().size() ); assertEquals( ValueMetaInterface.TYPE_STRING, rm.getValueMeta( 0 ).getType() ); assertEquals( BaseMessages.getString( PKG, "SearchResult.TransOrJob" ), rm.getValueMeta( 0 ).getName() ); assertEquals( ValueMetaInterface.TYPE_STRING, rm.getValueMeta( 1 ).getType() ); assertEquals( BaseMessages.getString( PKG, "SearchResult.StepDatabaseNotice" ), rm.getValueMeta( 1 ).getName() ); assertEquals( ValueMetaInterface.TYPE_STRING, rm.getValueMeta( 2 ).getType() ); assertEquals( BaseMessages.getString( PKG, "SearchResult.String" ), rm.getValueMeta( 2 ).getName() ); assertEquals( ValueMetaInterface.TYPE_STRING, rm.getValueMeta( 3 ).getType() ); assertEquals( BaseMessages.getString( PKG, "SearchResult.FieldName" ), rm.getValueMeta( 3 ).getName() ); } }