public TextFileInputReader( IBaseFileInputStepControl step, TextFileInputMeta meta, TextFileInputData data, FileObject file, LogChannelInterface log ) throws Exception { this.step = step; this.meta = meta; this.data = data; this.log = log; CompressionProvider provider = CompressionProviderFactory.getInstance().getCompressionProviderByName( meta.content.fileCompression ); if ( log.isDetailed() ) { log.logDetailed( "This is a compressed file being handled by the " + provider.getName() + " provider" ); } in = provider.createInputStream( KettleVFS.getInputStream( file ) ); in.nextEntry(); BufferedInputStream inStream = new BufferedInputStream( in, BUFFER_SIZE_INPUT_STREAM ); BOMDetector bom = new BOMDetector( inStream ); if ( bom.bomExist() ) { // if BOM exist, use it instead defined charset isr = new InputStreamReader( inStream, bom.getCharset() ); } else if ( meta.getEncoding() != null && meta.getEncoding().length() > 0 ) { isr = new InputStreamReader( inStream, meta.getEncoding() ); } else { isr = new InputStreamReader( inStream ); } String encoding = isr.getEncoding(); data.encodingType = EncodingType.guessEncodingType( encoding ); readInitial(); }
List<String> filterNames = new ArrayList<String>(); if ( !Utils.isEmpty( provider.getDefaultExtension() ) && !Utils.isEmpty( provider.getName() ) ) { filterExtensions.add( "*." + provider.getDefaultExtension() ); filterNames.add( provider.getName() + " files" );
@Test public void testCreateCoreProviders() { CompressionProvider provider = factory.createCompressionProviderInstance( "None" ); assertNotNull( provider ); assertTrue( provider.getClass().isAssignableFrom( NoneCompressionProvider.class ) ); assertEquals( "None", provider.getName() ); assertEquals( "No compression", provider.getDescription() ); provider = factory.createCompressionProviderInstance( "Zip" ); assertNotNull( provider ); assertTrue( provider.getClass().isAssignableFrom( ZIPCompressionProvider.class ) ); assertEquals( "Zip", provider.getName() ); assertEquals( "ZIP compression", provider.getDescription() ); provider = factory.createCompressionProviderInstance( "GZip" ); assertNotNull( provider ); assertTrue( provider.getClass().isAssignableFrom( GZIPCompressionProvider.class ) ); assertEquals( "GZip", provider.getName() ); assertEquals( "GZIP compression", provider.getDescription() ); provider = factory.createCompressionProviderInstance( "Snappy" ); assertNotNull( provider ); assertTrue( provider.getClass().isAssignableFrom( SnappyCompressionProvider.class ) ); assertEquals( "Snappy", provider.getName() ); assertEquals( "Snappy compression", provider.getDescription() ); provider = factory.createCompressionProviderInstance( "Hadoop-snappy" ); assertNotNull( provider ); assertTrue( provider.getClass().isAssignableFrom( HadoopSnappyCompressionProvider.class ) ); assertEquals( "Hadoop-snappy", provider.getName() ); assertEquals( "Hadoop Snappy compression", provider.getDescription() ); }
@Override public String[] getCompressionProviderNames() { ArrayList<String> providerNames = new ArrayList<String>(); List<PluginInterface> providers = getPlugins(); if ( providers != null ) { for ( PluginInterface plugin : providers ) { try { CompressionProvider provider = PluginRegistry.getInstance().loadClass( plugin, CompressionProvider.class ); if ( provider != null ) { providerNames.add( provider.getName() ); } } catch ( Exception e ) { // Do nothing here, if we can't load the provider, don't add it to the list } } } return providerNames.toArray( new String[providerNames.size()] ); }
@Override public InputStream getInputStream( final CsvInputAwareMeta meta ) { InputStream fileInputStream; CompressionInputStream inputStream = null; try { FileObject fileObject = meta.getHeaderFileObject( getTransMeta() ); fileInputStream = KettleVFS.getInputStream( fileObject ); CompressionProvider provider = CompressionProviderFactory.getInstance().createCompressionProviderInstance( ( (TextFileInputMeta) meta ).content.fileCompression ); inputStream = provider.createInputStream( fileInputStream ); } catch ( final Exception e ) { logError( BaseMessages.getString( "FileInputDialog.ErrorGettingFileDesc.DialogMessage" ), e ); } return inputStream; } }
logDetailed( "Opening output stream using provider: " + compressionProvider.getName() ); compressionOutputStream = compressionProvider.createOutputStream( fileOutputStream ); compressionOutputStream = compressionProvider.createOutputStream( fileOutputStream ); compressionOutputStream.addEntry( filename, environmentSubstitute( meta.getExtension() ) ); bufferedOutputStream = new BufferedOutputStream( compressionOutputStream, 5000 );
private CompressionProvider getCompressionProvider() throws KettleException { String compressionType = meta.getFileCompression(); if ( Utils.isEmpty( compressionType ) ) { compressionType = FILE_COMPRESSION_TYPE_NONE; } CompressionProvider compressionProvider = CompressionProviderFactory.getInstance().getCompressionProviderByName( compressionType ); if ( compressionProvider == null ) { throw new KettleException( "No compression provider found with name = " + compressionType ); } if ( !compressionProvider.supportsOutput() ) { throw new KettleException( "Compression provider " + compressionType + " does not support output streams!" ); } return compressionProvider; }
@Override public CompressionProvider getCompressionProviderByName( String name ) { if ( name == null ) { return null; } CompressionProvider foundProvider = null; List<PluginInterface> providers = getPlugins(); if ( providers != null ) { for ( PluginInterface plugin : providers ) { try { CompressionProvider provider = PluginRegistry.getInstance().loadClass( plugin, CompressionProvider.class ); if ( provider != null && name.equals( provider.getName() ) ) { foundProvider = provider; } } catch ( Exception e ) { // Do nothing here, if we can't load the provider, don't add it to the list } } } return foundProvider; }
f = provider.createInputStream( fi );
@Test public void getCompressionProvider() { assertEquals( provider.getName(), PROVIDER_NAME ); }
CompressionProviderFactory.getInstance().getCompressionProviderByName( meta.getFileCompression() ); data.in = provider.createInputStream( KettleVFS.getInputStream( data.file ) ); data.dataErrorLineHandler.handleFile( data.file ); data.in.nextEntry(); logDetailed( "This is a compressed file being handled by the " + provider.getName() + " provider" );
f = provider.createInputStream( fi );
@Test public void getCompressionProvider() { CompressionProvider provider = outStream.getCompressionProvider(); assertEquals( provider.getName(), PROVIDER_NAME ); }
inputStream = provider.createInputStream( fileInputStream );
@Test public void getCompressionProvider() { CompressionProvider provider = outStream.getCompressionProvider(); assertEquals( provider.getName(), PROVIDER_NAME ); }
f = provider.createInputStream( fi );
@Test public void getZIPCompressionProvider() { CompressionProvider provider = inStream.getCompressionProvider(); assertEquals( provider.getName(), PROVIDER_NAME ); }
inputStream = provider.createInputStream( fileInputStream );
@Test public void getCompressionProvider() { CompressionProvider provider = inStream.getCompressionProvider(); assertEquals( provider.getName(), PROVIDER_NAME ); }
@Test public void getZIPCompressionProvider() { CompressionProvider provider = inStream.getCompressionProvider(); assertEquals( provider.getName(), PROVIDER_NAME ); }