private KettleVFS() { fsm = new ConcurrentFileSystemManager(); try { fsm.setFilesCache( new WeakRefFilesCache() ); fsm.init(); } catch ( FileSystemException e ) { e.printStackTrace(); } // Install a shutdown hook to make sure that the file system manager is closed // This will clean up temporary files in vfs_cache Runtime.getRuntime().addShutdownHook( new Thread( new Runnable() { @Override public void run() { if ( fsm != null ) { try { fsm.close(); } catch ( Exception ignored ) { // Exceptions can be thrown due to a closed classloader } } } } ) ); }
public void reset() { defaultVariableSpace = new Variables(); defaultVariableSpace.initializeVariablesFrom( null ); fsm.close(); try { fsm.setFilesCache( new WeakRefFilesCache() ); fsm.init(); } catch ( FileSystemException ignored ) { } }
vfs.setReplicator(new UniqueFileReplicator(new File(System.getProperty("java.io.tmpdir")))); vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE); vfs.init(); return vfs;
vfs.setReplicator(new UniqueFileReplicator(cacheDir)); vfs.setCacheStrategy(CacheStrategy.ON_RESOLVE); vfs.init(); vfsInstances.add(new WeakReference<>(vfs)); return vfs;
/** * Creates a {@link FileSystemManager} instance. * @return a {@link FileSystemManager} instance. * @throws FileSystemException if an error occurs creating the manager. */ protected FileSystemManager createFileSystemManager() throws FileSystemException { FileSystemManager fileSystemManager = null; try { if (getFileSystemManagerClassName() == null) { fileSystemManager = new StandardFileSystemManager(); } else { final Class<?> mgrClass = Class.forName(getFileSystemManagerClassName()); fileSystemManager = (FileSystemManager) mgrClass.newInstance(); } if (fileSystemManager instanceof DefaultFileSystemManager) { ((DefaultFileSystemManager) fileSystemManager).init(); } } catch (final Exception e) { throw new FileSystemException( "Could not create file system manager of class: " + getFileSystemManagerClassName(), e); } return fileSystemManager; }
DefaultFileSystemManager currMan = new DefaultFileSystemManager(); currMan.addProvider(currSchemeStr, new AzFileProvider()); currMan.addProvider("file", new DefaultLocalFileProvider()); currMan.init(); currMan.resolveFile("file:///tmp" + + file.getAbsolutePath());
/** * Initializes this manager. Adds the providers and replicator. * * @throws FileSystemException if an error occurs. */ @Override public void init() throws FileSystemException { // Set the replicator and temporary file store (use the same component) final DefaultFileReplicator replicator = createDefaultFileReplicator(); setReplicator(new PrivilegedFileReplicator(replicator)); setTemporaryFileStore(replicator); if (configUri == null) { // Use default config final URL url = getClass().getResource(CONFIG_RESOURCE); FileSystemException.requireNonNull(url, "vfs.impl/find-config-file.error", CONFIG_RESOURCE); configUri = url; } configure(configUri); configurePlugins(); // Initialise super-class super.init(); }
/** * Initializes this manager. Adds the providers and replicator. * * @throws FileSystemException if an error occurs. */ @Override public void init() throws FileSystemException { // Set the replicator and temporary file store (use the same component) final DefaultFileReplicator replicator = createDefaultFileReplicator(); setReplicator(new PrivilegedFileReplicator(replicator)); setTemporaryFileStore(replicator); if (configUri == null) { // Use default config final URL url = getClass().getResource(CONFIG_RESOURCE); if (url == null) { throw new FileSystemException("vfs.impl/find-config-file.error", CONFIG_RESOURCE); } configUri = url; } configure(configUri); configurePlugins(); // Initialise super-class super.init(); }
/** * Initializes this manager. Adds the providers and replicator. * * @throws FileSystemException if an error occurs. */ @Override public void init() throws FileSystemException { // Set the replicator and temporary file store (use the same component) final DefaultFileReplicator replicator = createDefaultFileReplicator(); setReplicator(new PrivilegedFileReplicator(replicator)); setTemporaryFileStore(replicator); if (configUri == null) { // Use default config final URL url = getClass().getResource(CONFIG_RESOURCE); if (url == null) { throw new FileSystemException("vfs.impl/find-config-file.error", CONFIG_RESOURCE); } configUri = url; } configure(configUri); configurePlugins(); // Initialise super-class super.init(); }
@BeforeClass public static void setUp() throws Exception { Logger.getRootLogger().setLevel(Level.ERROR); // Put the MiniDFSCluster directory in the target directory final File data = new File("target/test/hdfstestdata").getAbsoluteFile(); data.mkdirs(); System.setProperty("test.build.data", data.toString()); FileUtils.cleanDirectory(data); // Setup HDFS conf = new Configuration(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, HDFS_URI); conf.set("hadoop.security.token.service.use_ip", "true"); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024 * 1024); // 1M blocksize setUmask(conf); cluster = new MiniDFSCluster(PORT, conf, 1, true, true, true, null, null, null, null); cluster.waitActive(); // Set up the VFS manager = new DefaultFileSystemManager(); manager.addProvider("hdfs", new HdfsFileProvider()); manager.init(); hdfs = cluster.getFileSystem(); }
@BeforeClass public static void setUp() throws Exception { Logger.getRootLogger().setLevel(Level.ERROR); // Put the MiniDFSCluster directory in the target directory final File data = new File("target/test/hdfstestdata").getAbsoluteFile(); data.mkdirs(); System.setProperty("test.build.data", data.toString()); FileUtils.cleanDirectory(data); // Setup HDFS conf = new Configuration(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, HDFS_URI); conf.set("hadoop.security.token.service.use_ip", "true"); conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024 * 1024); // 1M blocksize setUmask(conf); cluster = new MiniDFSCluster(PORT, conf, 1, true, true, true, null, null, null, null); cluster.waitActive(); // Set up the VFS manager = new DefaultFileSystemManager(); manager.addProvider("hdfs", new HdfsFileProvider()); manager.init(); hdfs = cluster.getFileSystem(); }
/** * JUnit Fixture: Prepare a simple FSM. * * @throws FileSystemException for runtime problems */ @Before public void setUp() throws FileSystemException { manager = new DefaultFileSystemManager(); final FileProvider fp = new DefaultLocalFileProvider(); manager.addProvider("file", fp); manager.init(); }
/** * JUnit Fixture: Prepare a simple FSM. * * @throws FileSystemException for runtime problems */ @Before public void setUp() throws FileSystemException { manager = new DefaultFileSystemManager(); final FileProvider fp = new DefaultLocalFileProvider(); manager.addProvider("file", fp); manager.init(); }
@Before public void setUp() throws Exception { manager = new DefaultFileSystemManager(); manager.addProvider("ram", new RamFileProvider()); manager.init(); // File Systems Options RamFileSystemConfigBuilder.getInstance().setMaxSize(zeroSizedFso, 0L); RamFileSystemConfigBuilder.getInstance().setMaxSize(smallSizedFso, 10L); }
@Before public void setUp() throws Exception { manager = new DefaultFileSystemManager(); manager.addProvider("ram", new RamFileProvider()); manager.init(); // File Systems Options RamFileSystemConfigBuilder.getInstance().setMaxSize(zeroSizedFso, 0L); RamFileSystemConfigBuilder.getInstance().setMaxSize(smallSizedFso, 10L); }
@Before public void setUp() throws Exception { manager = new DefaultFileSystemManager(); manager.addProvider("ram", new RamFileProvider()); manager.init(); // File Systems Options RamFileSystemConfigBuilder.getInstance().setMaxSize(zeroSizedFso, 0L); RamFileSystemConfigBuilder.getInstance().setMaxSize(smallSizedFso, 10L); }
/** * Test the on_call strategy */ public void testOnCallCache() throws Exception { final FileObject scratchFolder = getWriteFolder(); if (FileObjectUtils.isInstanceOf(getBaseFolder(), RamFileObject.class) || scratchFolder.getFileSystem() instanceof VirtualFileSystem) { // cant check ram filesystem as every manager holds its own ram filesystem data return; } scratchFolder.delete(Selectors.EXCLUDE_SELF); final DefaultFileSystemManager fs = createManager(); fs.setCacheStrategy(CacheStrategy.ON_CALL); fs.init(); final FileObject foBase2 = getBaseTestFolder(fs); final FileObject cachedFolder = foBase2.resolveFile(scratchFolder.getName().getPath()); FileObject[] fos = cachedFolder.getChildren(); assertContainsNot(fos, "file1.txt"); scratchFolder.resolveFile("file1.txt").createFile(); fos = cachedFolder.getChildren(); assertContains(fos, "file1.txt"); }
/** * Test the on_call strategy */ public void testOnCallCache() throws Exception { final FileObject scratchFolder = getWriteFolder(); if (FileObjectUtils.isInstanceOf(getBaseFolder(), RamFileObject.class) || scratchFolder.getFileSystem() instanceof VirtualFileSystem) { // cant check ram filesystem as every manager holds its own ram filesystem data return; } scratchFolder.delete(Selectors.EXCLUDE_SELF); final DefaultFileSystemManager fs = createManager(); fs.setCacheStrategy(CacheStrategy.ON_CALL); fs.init(); final FileObject foBase2 = getBaseTestFolder(fs); final FileObject cachedFolder = foBase2.resolveFile(scratchFolder.getName().getPath()); FileObject[] fos = cachedFolder.getChildren(); assertContainsNot(fos, "file1.txt"); scratchFolder.resolveFile("file1.txt").createFile(); fos = cachedFolder.getChildren(); assertContains(fos, "file1.txt"); }
/** * Test the on_resolve strategy */ public void testOnResolveCache() throws Exception { final FileObject scratchFolder = getWriteFolder(); if (FileObjectUtils.isInstanceOf(getBaseFolder(), RamFileObject.class) || scratchFolder.getFileSystem() instanceof VirtualFileSystem) { // cant check ram filesystem as every manager holds its own ram filesystem data return; } scratchFolder.delete(Selectors.EXCLUDE_SELF); final DefaultFileSystemManager fs = createManager(); fs.setCacheStrategy(CacheStrategy.ON_RESOLVE); fs.init(); final FileObject foBase2 = getBaseTestFolder(fs); FileObject cachedFolder = foBase2.resolveFile(scratchFolder.getName().getPath()); FileObject[] fos = cachedFolder.getChildren(); assertContainsNot(fos, "file1.txt"); scratchFolder.resolveFile("file1.txt").createFile(); fos = cachedFolder.getChildren(); assertContainsNot(fos, "file1.txt"); cachedFolder = foBase2.resolveFile(scratchFolder.getName().getPath()); fos = cachedFolder.getChildren(); assertContains(fos, "file1.txt"); }
/** * Test the manual cache strategy */ public void testManualCache() throws Exception { final FileObject scratchFolder = getWriteFolder(); if (FileObjectUtils.isInstanceOf(getBaseFolder(), RamFileObject.class) || scratchFolder.getFileSystem() instanceof VirtualFileSystem) { // cant check ram filesystem as every manager holds its own ram filesystem data return; } scratchFolder.delete(Selectors.EXCLUDE_SELF); final DefaultFileSystemManager fs = createManager(); fs.setCacheStrategy(CacheStrategy.MANUAL); fs.init(); final FileObject foBase2 = getBaseTestFolder(fs); final FileObject cachedFolder = foBase2.resolveFile(scratchFolder.getName().getPath()); FileObject[] fos = cachedFolder.getChildren(); assertContainsNot(fos, "file1.txt"); scratchFolder.resolveFile("file1.txt").createFile(); fos = cachedFolder.getChildren(); assertContainsNot(fos, "file1.txt"); cachedFolder.refresh(); fos = cachedFolder.getChildren(); assertContains(fos, "file1.txt"); }