public String getXML() { StringBuilder retval = new StringBuilder( 300 ); retval.append( " " ).append( XMLHandler.addTagValue( "pass_all_rows", passAllRows ) ); retval.append( " " ).append( XMLHandler.addTagValue( "directory", directory ) ); retval.append( " " ).append( XMLHandler.addTagValue( "prefix", prefix ) ); retval.append( " " ).append( XMLHandler.addTagValue( "cache_size", cacheSize ) ); retval.append( " " ).append( XMLHandler.addTagValue( "compress", compressFiles ) ); parentStepMeta.getParentTransMeta().getNamedClusterEmbedManager().registerUrl( directory ); return retval.toString(); }
@VisibleForTesting protected void registerUrlWithDirectory() { parentStepMeta.getParentTransMeta().getNamedClusterEmbedManager().registerUrl( directory ); }
public String getXML() { StringBuilder retval = new StringBuilder( 300 ); retval.append( " " ).append( XMLHandler.addTagValue( "rownum", includeRowNumber ) ); retval.append( " " ).append( XMLHandler.addTagValue( "foldername_dynamic", isFoldernameDynamic ) ); retval.append( " " ).append( XMLHandler.addTagValue( "rownum_field", rowNumberField ) ); retval.append( " " ).append( XMLHandler.addTagValue( "foldername_field", dynamicFoldernameField ) ); retval.append( " " ).append( XMLHandler.addTagValue( "limit", rowLimit ) ); retval.append( " <file>" ).append( Const.CR ); for ( int i = 0; i < folderName.length; i++ ) { retval.append( " " ).append( XMLHandler.addTagValue( "name", folderName[i] ) ); retval.append( " " ).append( XMLHandler.addTagValue( "file_required", folderRequired[i] ) ); parentStepMeta.getParentTransMeta().getNamedClusterEmbedManager().registerUrl( folderName[i] ); } retval.append( " </file>" ).append( Const.CR ); return retval.toString(); }
public String getXML() { StringBuilder retval = new StringBuilder( 300 ); retval.append( " " ).append( XMLHandler.addTagValue( "files_count", includeFilesCount ) ); retval.append( " " ).append( XMLHandler.addTagValue( "files_count_fieldname", filesCountFieldName ) ); retval.append( " " ).append( XMLHandler.addTagValue( "rows_count_fieldname", rowsCountFieldName ) ); retval.append( " " ).append( XMLHandler.addTagValue( "rowseparator_format", RowSeparator_format ) ); retval.append( " " ).append( XMLHandler.addTagValue( "row_separator", RowSeparator ) ); retval.append( " " ).append( XMLHandler.addTagValue( "isaddresult", isaddresult ) ); retval.append( " " ).append( XMLHandler.addTagValue( "filefield", filefield ) ); retval.append( " " ).append( XMLHandler.addTagValue( "filename_Field", outputFilenameField ) ); retval.append( " " ).append( XMLHandler.addTagValue( "smartCount", smartCount ) ); retval.append( " <file>" ).append( Const.CR ); for ( int i = 0; i < fileName.length; i++ ) { retval.append( " " ).append( XMLHandler.addTagValue( "name", fileName[i] ) ); retval.append( " " ).append( XMLHandler.addTagValue( "filemask", fileMask[i] ) ); retval.append( " " ).append( XMLHandler.addTagValue( "exclude_filemask", excludeFileMask[i] ) ); retval.append( " " ).append( XMLHandler.addTagValue( "file_required", fileRequired[i] ) ); retval.append( " " ).append( XMLHandler.addTagValue( "include_subfolders", includeSubFolders[i] ) ); parentStepMeta.getParentTransMeta().getNamedClusterEmbedManager().registerUrl( fileName[i] ); } retval.append( " </file>" ).append( Const.CR ); return retval.toString(); }
public String getXML() { StringBuilder retval = new StringBuilder(); retval.append( " " + XMLHandler.addTagValue( "sourcefilenamefield", sourcefilenamefield ) ); retval.append( " " + XMLHandler.addTagValue( "targetfilenamefield", targetfilenamefield ) ); retval.append( " " + XMLHandler.addTagValue( "baseFolderField", baseFolderField ) ); retval.append( " " + XMLHandler.addTagValue( "operation_type", getOperationTypeCode( operationType ) ) ); retval.append( " " ).append( XMLHandler.addTagValue( "addresultfilenames", addresultfilenames ) ); retval.append( " " ).append( XMLHandler.addTagValue( "overwritezipentry", overwritezipentry ) ); retval.append( " " ).append( XMLHandler.addTagValue( "createparentfolder", createparentfolder ) ); retval.append( " " ).append( XMLHandler.addTagValue( "keepsourcefolder", keepsourcefolder ) ); retval.append( " " + XMLHandler.addTagValue( "movetofolderfield", movetofolderfield ) ); parentStepMeta.getParentTransMeta().getNamedClusterEmbedManager().registerUrl( sourcefilenamefield ); parentStepMeta.getParentTransMeta().getNamedClusterEmbedManager().registerUrl( targetfilenamefield ); parentStepMeta.getParentTransMeta().getNamedClusterEmbedManager().registerUrl( baseFolderField ); return retval.toString(); }
protected void saveFileOptions( StringBuilder retval ) { if ( parentStepMeta != null && parentStepMeta.getParentTransMeta() != null ) { parentStepMeta.getParentTransMeta().getNamedClusterEmbedManager().registerUrl( fileName ); } saveSource( retval, fileName ); retval.append( " " ).append( XMLHandler.addTagValue( "servlet_output", servletOutput ) ); retval.append( " " ).append( XMLHandler.addTagValue( "do_not_open_new_file_init", doNotOpenNewFileInit ) ); retval.append( " " ).append( XMLHandler.addTagValue( "extention", extension ) ); retval.append( " " ).append( XMLHandler.addTagValue( "append", fileAppended ) ); retval.append( " " ).append( XMLHandler.addTagValue( "split", stepNrInFilename ) ); retval.append( " " ).append( XMLHandler.addTagValue( "haspartno", partNrInFilename ) ); retval.append( " " ).append( XMLHandler.addTagValue( "add_date", dateInFilename ) ); retval.append( " " ).append( XMLHandler.addTagValue( "add_time", timeInFilename ) ); retval.append( " " ).append( XMLHandler.addTagValue( "SpecifyFormat", isSpecifyingFormat() ) ); retval.append( " " ).append( XMLHandler.addTagValue( "date_time_format", getDateTimeFormat() ) ); retval.append( " " ).append( XMLHandler.addTagValue( "add_to_result_filenames", addToResultFilenames ) ); retval.append( " " ).append( XMLHandler.addTagValue( "pad", padded ) ); retval.append( " " ).append( XMLHandler.addTagValue( "fast_dump", fastDump ) ); retval.append( " " ).append( XMLHandler.addTagValue( "splitevery", splitEveryRows ) ); }
public boolean init( StepMetaInterface smi, StepDataInterface sdi ) { meta = (ZipFileMeta) smi; data = (ZipFileData) sdi; if ( super.init( smi, sdi ) ) { //Set Embedded NamedCluter MetatStore Provider Key so that it can be passed to VFS if ( getTransMeta().getNamedClusterEmbedManager() != null ) { getTransMeta().getNamedClusterEmbedManager() .passEmbeddedMetastoreKey( this, getTransMeta().getEmbeddedMetastoreProviderKey() ); } return true; } return false; }
@Override public String getXML() { StringBuilder retval = new StringBuilder( 300 ); retval.append( " <filter>" ).append( Const.CR ); retval.append( " " ).append( XMLHandler.addTagValue( "filterfiletype", fileTypeFilter.toString() ) ); retval.append( " </filter>" ).append( Const.CR ); retval.append( " " ).append( XMLHandler.addTagValue( "doNotFailIfNoFile", doNotFailIfNoFile ) ); retval.append( " " ).append( XMLHandler.addTagValue( "rownum", includeRowNumber ) ); retval.append( " " ).append( XMLHandler.addTagValue( "isaddresult", isaddresult ) ); retval.append( " " ).append( XMLHandler.addTagValue( "filefield", filefield ) ); retval.append( " " ).append( XMLHandler.addTagValue( "rownum_field", rowNumberField ) ); retval.append( " " ).append( XMLHandler.addTagValue( "filename_Field", dynamicFilenameField ) ); retval.append( " " ).append( XMLHandler.addTagValue( "wildcard_Field", dynamicWildcardField ) ); retval .append( " " ).append( XMLHandler.addTagValue( "exclude_wildcard_Field", dynamicExcludeWildcardField ) ); retval.append( " " ).append( XMLHandler.addTagValue( "dynamic_include_subfolders", dynamicIncludeSubFolders ) ); retval.append( " " ).append( XMLHandler.addTagValue( "limit", rowLimit ) ); retval.append( " <file>" ).append( Const.CR ); for ( int i = 0; i < fileName.length; i++ ) { retval.append( " " ).append( XMLHandler.addTagValue( "name", fileName[i] ) ); retval.append( " " ).append( XMLHandler.addTagValue( "filemask", fileMask[i] ) ); retval.append( " " ).append( XMLHandler.addTagValue( "exclude_filemask", excludeFileMask[i] ) ); retval.append( " " ).append( XMLHandler.addTagValue( "file_required", fileRequired[i] ) ); retval.append( " " ).append( XMLHandler.addTagValue( "include_subfolders", includeSubFolders[i] ) ); parentStepMeta.getParentTransMeta().getNamedClusterEmbedManager().registerUrl( fileName[i] ); } retval.append( " </file>" ).append( Const.CR ); return retval.toString(); }
public boolean init( StepMetaInterface smi, StepDataInterface sdi ) { meta = (BlockingStepMeta) smi; data = (BlockingStepData) sdi; if ( super.init( smi, sdi ) ) { // Set Embedded NamedCluter MetatStore Provider Key so that it can be passed to VFS if ( getTransMeta().getNamedClusterEmbedManager() != null ) { getTransMeta().getNamedClusterEmbedManager().passEmbeddedMetastoreKey( getTransMeta(), getTransMeta().getEmbeddedMetastoreProviderKey() ); } // Add init code here. return true; } return false; }
public T createMeta() { try { T meta = clazz.newInstance(); if ( meta instanceof BaseStepMeta ) { StepMeta mockParentStepMeta = mock( StepMeta.class ); ( (BaseStepMeta) meta ).setParentStepMeta( mockParentStepMeta ); TransMeta mockTransMeta = mock( TransMeta.class ); NamedClusterEmbedManager embedManager = mock( NamedClusterEmbedManager.class ); when( mockParentStepMeta.getParentTransMeta() ).thenReturn( mockTransMeta ); when( mockTransMeta.getNamedClusterEmbedManager() ).thenReturn( embedManager ); } return meta; } catch ( Exception e ) { throw new RuntimeException( "Unable to create meta of class " + clazz.getCanonicalName(), e ); } }
@Before public void setUp() throws Exception { NamedClusterEmbedManager manager = mock( NamedClusterEmbedManager.class ); TransMeta parentTransMeta = mock( TransMeta.class ); doReturn( manager ).when( parentTransMeta ).getNamedClusterEmbedManager(); StepMeta parentStepMeta = mock( StepMeta.class ); doReturn( parentTransMeta ).when( parentStepMeta ).getParentTransMeta(); inputMeta = new TextFileInputMeta(); inputMeta.setParentStepMeta( parentStepMeta ); inputMeta = spy( inputMeta ); variableSpace = mock( VariableSpace.class ); doReturn( "<def>" ).when( variableSpace ).environmentSubstitute( anyString() ); doReturn( FILE_NAME_VALID_PATH ).when( variableSpace ).environmentSubstitute( FILE_NAME_VALID_PATH ); FileObject mockedFileObject = mock( FileObject.class ); doReturn( mockedFileObject ).when( inputMeta ).getFileObject( anyString(), eq( variableSpace ) ); }
retval.append( " " ).append( XMLHandler.addTagValue( "file_required", fileRequired[i] ) ); retval.append( " " ).append( XMLHandler.addTagValue( "include_subfolders", includeSubFolders[i] ) ); parentStepMeta.getParentTransMeta().getNamedClusterEmbedManager().registerUrl( fileName[i] );
public boolean init( StepMetaInterface smi, StepDataInterface sdi ) { meta = (GetSubFoldersMeta) smi; data = (GetSubFoldersData) sdi; if ( super.init( smi, sdi ) ) { //Set Embedded NamedCluter MetatStore Provider Key so that it can be passed to VFS if ( getTransMeta().getNamedClusterEmbedManager() != null ) { getTransMeta().getNamedClusterEmbedManager() .passEmbeddedMetastoreKey( this, getTransMeta().getEmbeddedMetastoreProviderKey() ); } try { data.filessize = 0; data.rownr = 1L; data.filenr = 0; data.totalpreviousfields = 0; } catch ( Exception e ) { logError( "Error initializing step: " + e.toString() ); logError( Const.getStackTracker( e ) ); return false; } return true; } return false; }
@Test public void testLoadAndGetXml() throws Exception { ZipFileMeta zipFileMeta = new ZipFileMeta(); Node stepnode = getTestNode(); DatabaseMeta dbMeta = mock( DatabaseMeta.class ); IMetaStore metaStore = mock( IMetaStore.class ); StepMeta mockParentStepMeta = mock( StepMeta.class ); zipFileMeta.setParentStepMeta( mockParentStepMeta ); TransMeta mockTransMeta = mock( TransMeta.class ); NamedClusterEmbedManager embedManager = mock( NamedClusterEmbedManager.class ); when( mockParentStepMeta.getParentTransMeta() ).thenReturn( mockTransMeta ); when( mockTransMeta.getNamedClusterEmbedManager() ).thenReturn( embedManager ); zipFileMeta.loadXML( stepnode, Collections.singletonList( dbMeta ), metaStore ); assertXmlOutputMeta( zipFileMeta ); }
@Override public boolean init( StepMetaInterface smi, StepDataInterface sdi ) { meta = (SortRowsMeta) smi; data = (SortRowsData) sdi; if ( !super.init( smi, sdi ) ) { return false; } //Set Embedded NamedCluter MetatStore Provider Key so that it can be passed to VFS if ( getTransMeta().getNamedClusterEmbedManager() != null ) { getTransMeta().getNamedClusterEmbedManager() .passEmbeddedMetastoreKey( getTransMeta(), getTransMeta().getEmbeddedMetastoreProviderKey() ); } data.sortSize = Const.toInt( environmentSubstitute( meta.getSortSize() ), -1 ); data.freeMemoryPctLimit = Const.toInt( meta.getFreeMemoryLimit(), -1 ); if ( data.sortSize <= 0 && data.freeMemoryPctLimit <= 0 ) { // Prefer the memory limit as it should never fail // data.freeMemoryPctLimit = 25; } // In memory buffer // data.buffer = new ArrayList<Object[]>( 5000 ); // Buffer for reading from disk // data.rowbuffer = new ArrayList<Object[]>( 5000 ); data.compressFiles = getBooleanValueOfVariable( meta.getCompressFilesVariable(), meta.getCompressFiles() ); data.tempRows = new ArrayList<RowTempFile>(); data.minSortSize = 5000; return true; }
public boolean init( StepMetaInterface smi, StepDataInterface sdi ) { meta = (PropertyInputMeta) smi; data = (PropertyInputData) sdi; if ( super.init( smi, sdi ) ) { //Set Embedded NamedCluter MetatStore Provider Key so that it can be passed to VFS if ( getTransMeta().getNamedClusterEmbedManager() != null ) { getTransMeta().getNamedClusterEmbedManager() .passEmbeddedMetastoreKey( this, getTransMeta().getEmbeddedMetastoreProviderKey() ); } String realEncoding = environmentSubstitute( meta.getEncoding() ); if ( !Utils.isEmpty( realEncoding ) ) { data.realEncoding = realEncoding; } String realSection = environmentSubstitute( meta.getSection() ); if ( !Utils.isEmpty( realSection ) ) { data.realSection = realSection; } data.propfiles = ( PropertyInputMeta.getFileTypeByDesc( meta.getFileType() ) == PropertyInputMeta.FILE_TYPE_PROPERTY ); data.rownr = 1L; data.totalpreviousfields = 0; return true; } return false; }
if ( getTransMeta().getNamedClusterEmbedManager() != null ) { getTransMeta().getNamedClusterEmbedManager() .passEmbeddedMetastoreKey( this, getTransMeta().getEmbeddedMetastoreProviderKey() );
if ( getTransMeta().getNamedClusterEmbedManager() != null ) { getTransMeta().getNamedClusterEmbedManager() .passEmbeddedMetastoreKey( this, getTransMeta().getEmbeddedMetastoreProviderKey() );
if ( getTransMeta().getNamedClusterEmbedManager() != null ) { getTransMeta().getNamedClusterEmbedManager() .passEmbeddedMetastoreKey( this, getTransMeta().getEmbeddedMetastoreProviderKey() );
if ( getTransMeta().getNamedClusterEmbedManager() != null ) { getTransMeta().getNamedClusterEmbedManager() .passEmbeddedMetastoreKey( getTransMeta(), getTransMeta().getEmbeddedMetastoreProviderKey() );