public static Object sum( ValueMetaInterface metaA, Object dataA, ValueMetaInterface metaB, Object dataB ) throws KettleValueException { if ( dataA == null && dataB == null ) { return null; } if ( dataA == null && dataB != null ) { Object value = metaA.convertData( metaB, dataB ); metaA.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); return value; } if ( dataA != null && dataB == null ) { return dataA; } return plus( metaA, dataA, metaB, dataB ); }
private void initGroupMeta( RowMetaInterface previousRowMeta ) throws KettleValueException { data.groupMeta = new RowMeta(); data.entryMeta = new RowMeta(); for ( int i = 0; i < data.groupnrs.length; i++ ) { ValueMetaInterface valueMeta = previousRowMeta.getValueMeta( data.groupnrs[i] ); data.groupMeta.addValueMeta( valueMeta ); ValueMetaInterface normalMeta = valueMeta.clone(); normalMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); } return; }
private Database createVirtualDb( DatabaseMeta meta ) throws Exception { ResultSet rs = mock( ResultSet.class ); when( rs.getMetaData() ).thenReturn( mock( ResultSetMetaData.class ) ); PreparedStatement ps = mock( PreparedStatement.class ); when( ps.executeQuery() ).thenReturn( rs ); Connection connection = mock( Connection.class ); when( connection.prepareStatement( anyString() ) ).thenReturn( ps ); Database db = new Database( mock( LoggingObjectInterface.class ), meta ); db.setConnection( connection ); db = spy( db ); doNothing().when( db ).normalConnect( anyString() ); ValueMetaInterface binary = new ValueMetaString( BINARY_FIELD ); binary.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING ); ValueMetaInterface id = new ValueMetaInteger( ID_FIELD ); RowMetaInterface metaByQuerying = new RowMeta(); metaByQuerying.addValueMeta( binary ); metaByQuerying.addValueMeta( id ); doReturn( metaByQuerying ).when( db ).getTableFields( anyString() ); doReturn( metaByQuerying ).when( db ).getTableFieldsMeta( anyString(), anyString() ); return db; }
@Test public void testSumWithNullValues() throws Exception { ValueMetaInterface metaA = new ValueMetaInteger(); metaA.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); ValueMetaInterface metaB = new ValueMetaInteger(); metaA.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); assertNull( ValueDataUtil.sum( metaA, null, metaB, null ) ); Long valueB = new Long( 2 ); ValueDataUtil.sum( metaA, null, metaB, valueB ); }
private Object[] processRow( RowMetaInterface rowMeta, Object[] row ) throws KettleException { Object[] RowData = new Object[data.outputRowMeta.size()]; // Copy the input fields. System.arraycopy( row, 0, RowData, 0, rowMeta.size() ); int j = 0; // Index into "new fields" area, past the first {data.inputFieldsNr} records for ( int i = 0; i < data.nrFieldsInStream; i++ ) { if ( data.inStreamNrs[i] >= 0 ) { // Get source value String value = getInputRowMeta().getString( row, data.inStreamNrs[i] ); // Apply String operations and return result value value = processString( value, data.trimOperators[i], data.lowerUpperOperators[i], data.padType[i], data.padChar[i], data.padLen[i], data.initCap[i], data.maskHTML[i], data.digits[i], data.removeSpecialCharacters[i] ); if ( Utils.isEmpty( data.outStreamNrs[i] ) ) { // Update field RowData[data.inStreamNrs[i]] = value; data.outputRowMeta.getValueMeta( data.inStreamNrs[i] ) .setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); } else { // create a new Field RowData[data.inputFieldsNr + j] = value; j++; } } } return RowData; }
@Test public void testSumConvertingStorageTypeToNormal() throws Exception { ValueMetaInterface metaA = mock( ValueMetaInteger.class ); metaA.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING ); ValueMetaInterface metaB = new ValueMetaInteger(); metaB.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING ); Object valueB = "2"; when( metaA.convertData( metaB, valueB ) ).thenAnswer( new Answer<Long>() { @Override public Long answer( InvocationOnMock invocation ) throws Throwable { return new Long( 2 ); } } ); Object returnValue = ValueDataUtil.sum( metaA, null, metaB, valueB ); verify( metaA ).convertData( metaB, valueB ); assertEquals( 2L, returnValue ); assertEquals( metaA.getStorageType(), ValueMetaInterface.STORAGE_TYPE_NORMAL ); }
public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException { for ( int i = 0; i < fieldOutStream.length; i++ ) { ValueMetaInterface v; if ( !Utils.isEmpty( fieldOutStream[i] ) ) { v = new ValueMetaString( space.environmentSubstitute( fieldOutStream[i] ) ); v.setLength( 100, -1 ); v.setOrigin( name ); inputRowMeta.addValueMeta( v ); } else { v = inputRowMeta.searchValueMeta( fieldInStream[i] ); if ( v == null ) { continue; } v.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); } } }
@Test public void addToAggregateBinaryData() throws Exception { MemoryGroupByMeta memoryGroupByMeta = Mockito.spy( meta ); memoryGroupByMeta.setAggregateType( new int[] { MemoryGroupByMeta.TYPE_GROUP_COUNT_DISTINCT } ); when( mockHelper.stepMeta.getStepMetaInterface() ).thenReturn( memoryGroupByMeta ); vmi.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); vmi.setStorageMetadata( new ValueMetaString() ); aggregate.counts = new long[] { 0L }; Mockito.doReturn( new String[] { "test" } ).when( memoryGroupByMeta ).getSubjectField(); aggregate.agg = new Object[] { new byte[0] }; step = new MemoryGroupBy( mockHelper.stepMeta, data, 0, mockHelper.transMeta, mockHelper.trans ); String binaryData0 = "11011"; String binaryData1 = "01011"; step.addToAggregate( new Object[] { binaryData0.getBytes() } ); step.addToAggregate( new Object[] { binaryData1.getBytes() } ); Object[] distinctObjs = data.map.get( getHashEntry() ).distinctObjs[0].toArray(); Assert.assertEquals( binaryData0, distinctObjs[1] ); Assert.assertEquals( binaryData1, distinctObjs[0] ); }
private void updateField( Object[] r ) throws Exception { // Loop through fields for ( int i = 0; i < data.getFieldnr(); i++ ) { // DO CONVERSION OF THE DEFAULT VALUE ... // Entered by user ValueMetaInterface targetValueMeta = data.getOutputRowMeta().getValueMeta( data.getFieldnrs()[i] ); ValueMetaInterface sourceValueMeta = data.getConvertRowMeta().getValueMeta( data.getFieldnrs()[i] ); if ( !Utils.isEmpty( meta.getField( i ).getReplaceMask() ) ) { sourceValueMeta.setConversionMask( meta.getField( i ).getReplaceMask() ); } sourceValueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); r[data.getFieldnrs()[i]] = targetValueMeta.convertData( sourceValueMeta, data.getRealReplaceByValues()[i] ); targetValueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); } }
valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING );
@Test public void addToAggregateLazyConversionMinTest() throws Exception { vmi.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING ); vmi.setStorageMetadata( new ValueMetaString() ); aggregate.agg = new Object[] { new byte[0] }; byte[] bytes = { 51 }; step.addToAggregate( new Object[] { bytes } ); Aggregate result = data.map.get( getHashEntry() ); Assert.assertEquals( "Returns non-null value", bytes, result.agg[0] ); }
@Test public void testGetFields() throws Exception { TableInputMetaHandler meta = new TableInputMetaHandler(); meta.setLazyConversionActive( true ); DatabaseMeta dbMeta = mock( DatabaseMeta.class ); meta.setDatabaseMeta( dbMeta ); Database mockDB = meta.getDatabase(); when( mockDB.getQueryFields( anyString(), anyBoolean() ) ).thenReturn( createMockFields() ); RowMetaInterface expectedRowMeta = new RowMeta(); ValueMetaInterface valueMeta = new ValueMetaString( "field1" ); valueMeta.setStorageMetadata( new ValueMetaString( "field1" ) ); valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING ); expectedRowMeta.addValueMeta( valueMeta ); VariableSpace space = mock( VariableSpace.class ); RowMetaInterface rowMetaInterface = new RowMeta(); meta.getFields( rowMetaInterface, "TABLE_INPUT_META", null, null, space, null, null ); assertEquals( expectedRowMeta.toString(), rowMetaInterface.toString() ); }
public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException { int nrFields = fieldInStream == null ? 0 : fieldInStream.length; for ( int i = 0; i < nrFields; i++ ) { String fieldName = space.environmentSubstitute( fieldOutStream[i] ); ValueMetaInterface valueMeta; if ( !Utils.isEmpty( fieldOutStream[i] ) ) { // We have a new field valueMeta = new ValueMetaString( fieldName ); valueMeta.setOrigin( name ); //set encoding to new field from source field http://jira.pentaho.com/browse/PDI-11839 ValueMetaInterface sourceField = inputRowMeta.searchValueMeta( fieldInStream[i] ); if ( sourceField != null ) { valueMeta.setStringEncoding( sourceField.getStringEncoding() ); } inputRowMeta.addValueMeta( valueMeta ); } else { valueMeta = inputRowMeta.searchValueMeta( fieldInStream[i] ); if ( valueMeta == null ) { continue; } valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); } } }
@Override public void getFields( RowMetaInterface r, String origin, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore ) { ValueMetaInterface extra = null; if ( !Utils.isEmpty( getOutputField() ) ) { extra = new ValueMetaString( space.environmentSubstitute( getOutputField() ) ); extra.setOrigin( origin ); r.addValueMeta( extra ); } else { if ( !Utils.isEmpty( getInputField() ) ) { extra = r.searchValueMeta( space.environmentSubstitute( getInputField() ) ); } } if ( extra != null ) { extra.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); } }
@Test public void testGetFields() throws Exception { StringOperationsMeta meta = new StringOperationsMeta(); meta.allocate( 1 ); meta.setFieldInStream( new String[] { "field1" } ); RowMetaInterface rowMetaInterface = new RowMeta(); ValueMetaInterface valueMeta = new ValueMetaString( "field1" ); valueMeta.setStorageMetadata( new ValueMetaString( "field1" ) ); valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING ); rowMetaInterface.addValueMeta( valueMeta ); VariableSpace space = mock( VariableSpace.class ); meta.getFields( rowMetaInterface, "STRING_OPERATIONS", null, null, space, null, null ); RowMetaInterface expectedRowMeta = new RowMeta(); expectedRowMeta.addValueMeta( new ValueMetaString( "field1" ) ); assertEquals( expectedRowMeta.toString(), rowMetaInterface.toString() ); } }
public boolean processRow( StepMetaInterface smi, StepDataInterface sdi ) throws KettleException { meta = (FixedInputMeta) smi; data = (FixedInputData) sdi; if ( first ) { first = false; data.outputRowMeta = new RowMeta(); meta.getFields( data.outputRowMeta, getStepname(), null, null, this, repository, metaStore ); // The conversion logic for when the lazy conversion is turned of is simple: // Pretend it's a lazy conversion object anyway and get the native type during conversion. // data.convertRowMeta = data.outputRowMeta.clone(); for ( ValueMetaInterface valueMeta : data.convertRowMeta.getValueMetaList() ) { valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING ); } if ( meta.isHeaderPresent() ) { readOneRow( false ); // skip this row. } } Object[] outputRowData = readOneRow( true ); if ( outputRowData == null ) { // no more input to be expected... setOutputDone(); return false; } putRow( data.outputRowMeta, outputRowData ); // copy row to possible alternate rowset(s). if ( checkFeedback( getLinesInput() ) ) { logBasic( BaseMessages.getString( PKG, "FixedInput.Log.LineNumber", Long.toString( getLinesInput() ) ) ); } return true; }
@Override public void getFields( RowMetaInterface inputRowMeta, String name, RowMetaInterface[] info, StepMeta nextStep, VariableSpace space, Repository repository, IMetaStore metaStore ) throws KettleStepException { // Add new field? for ( int i = 0; i < fieldOutStream.length; i++ ) { ValueMetaInterface v; String outputField = space.environmentSubstitute( fieldOutStream[i] ); if ( !Utils.isEmpty( outputField ) ) { // Add a new field v = new ValueMetaString( outputField ); v.setLength( 100, -1 ); v.setOrigin( name ); inputRowMeta.addValueMeta( v ); } else { v = inputRowMeta.searchValueMeta( fieldInStream[i] ); if ( v == null ) { continue; } v.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL ); int paddingType = getPaddingType()[i]; if ( paddingType == PADDING_LEFT || paddingType == PADDING_RIGHT ) { int padLen = Const.toInt( space.environmentSubstitute( getPadLen()[i] ), 0 ); if ( padLen > v.getLength() ) { // alter meta data v.setLength( padLen ); } } } } }
valueMeta.setStringEncoding( space.environmentSubstitute( encoding ) ); if ( lazyConversionActive ) { valueMeta.setStorageType( ValueMetaInterface.STORAGE_TYPE_BINARY_STRING ); storageMetadata.setStorageType( ValueMetaInterface.STORAGE_TYPE_NORMAL );
ValueMetaInterface valueMeta = new ValueMetaString( "field1" ); valueMeta.setStorageMetadata( new ValueMetaString( "field1" ) ); valueMeta.setStorageType( ValueMetaInterface.TYPE_STRING ); rowMetaInterface.addValueMeta( valueMeta ); when( lookupRowSet.getRowMeta() ).thenReturn( rowMetaInterface );
public static void cloneInfo( ValueMetaInterface source, ValueMetaInterface target ) throws KettlePluginException { target.setConversionMask( source.getConversionMask() ); target.setDecimalSymbol( source.getDecimalSymbol() ); target.setGroupingSymbol( source.getGroupingSymbol() ); target.setStorageType( source.getStorageType() ); if ( source.getStorageMetadata() != null ) { target.setStorageMetadata( cloneValueMeta( source.getStorageMetadata(), source .getStorageMetadata().getType() ) ); } target.setStringEncoding( source.getStringEncoding() ); target.setTrimType( source.getTrimType() ); target.setDateFormatLenient( source.isDateFormatLenient() ); target.setDateFormatLocale( source.getDateFormatLocale() ); target.setDateFormatTimeZone( source.getDateFormatTimeZone() ); target.setLenientStringToNumber( source.isLenientStringToNumber() ); target.setLargeTextField( source.isLargeTextField() ); target.setComments( source.getComments() ); target.setCaseInsensitive( source.isCaseInsensitive() ); target.setCollatorDisabled( source.isCollatorDisabled() ); target.setCollatorStrength( source.getCollatorStrength() ); target.setIndex( source.getIndex() ); target.setOrigin( source.getOrigin() ); target.setOriginalAutoIncrement( source.isOriginalAutoIncrement() ); target.setOriginalColumnType( source.getOriginalColumnType() ); target.setOriginalColumnTypeName( source.getOriginalColumnTypeName() ); target.setOriginalNullable( source.isOriginalNullable() ); target.setOriginalPrecision( source.getOriginalPrecision() ); target.setOriginalScale( source.getOriginalScale() ); target.setOriginalSigned( source.isOriginalSigned() ); }