private void checkLocalFunctionResources(Hive db, List<ResourceUri> resources) throws HiveException { // If this is a non-local warehouse, then adding resources from the local filesystem // may mean that other clients will not be able to access the resources. // So disallow resources from local filesystem in this case. if (resources != null && resources.size() > 0) { try { String localFsScheme = FileSystem.getLocal(db.getConf()).getUri().getScheme(); String configuredFsScheme = FileSystem.get(db.getConf()).getUri().getScheme(); if (configuredFsScheme.equals(localFsScheme)) { // Configured warehouse FS is local, don't need to bother checking. return; } for (ResourceUri res : resources) { String resUri = res.getUri(); if (ResourceDownloader.isFileUri(resUri)) { throw new HiveException("Hive warehouse is non-local, but " + res.getUri() + " specifies file on local filesystem. " + "Resources on non-local warehouse should specify a non-local scheme/path"); } } } catch (HiveException e) { throw e; } catch (Exception e) { LOG.error("Exception caught in checkLocalFunctionResources", e); throw new HiveException(e); } } }
private void checkLocalFunctionResources(Hive db, List<ResourceUri> resources) throws HiveException { // If this is a non-local warehouse, then adding resources from the local filesystem // may mean that other clients will not be able to access the resources. // So disallow resources from local filesystem in this case. if (resources != null && resources.size() > 0) { try { String localFsScheme = FileSystem.getLocal(db.getConf()).getUri().getScheme(); String configuredFsScheme = FileSystem.get(db.getConf()).getUri().getScheme(); if (configuredFsScheme.equals(localFsScheme)) { // Configured warehouse FS is local, don't need to bother checking. return; } for (ResourceUri res : resources) { String resUri = res.getUri(); if (ResourceDownloader.isFileUri(resUri)) { throw new HiveException("Hive warehouse is non-local, but " + res.getUri() + " specifies file on local filesystem. " + "Resources on non-local warehouse should specify a non-local scheme/path"); } } } catch (HiveException e) { throw e; } catch (Exception e) { LOG.error("Exception caught in checkLocalFunctionResources", e); throw new HiveException(e); } } }
/** * Sets up a new path in test filesystem to be used by tests. */ private Path getNewDataTestDirOnTestFS() throws IOException { //The file system can be either local, mini dfs, or if the configuration //is supplied externally, it can be an external cluster FS. If it is a local //file system, the tests should use getBaseTestDir, otherwise, we can use //the working directory, and create a unique sub dir there FileSystem fs = getTestFileSystem(); Path newDataTestDir; String randomStr = getRandomUUID().toString(); if (fs.getUri().getScheme().equals(FileSystem.getLocal(conf).getUri().getScheme())) { newDataTestDir = new Path(getDataTestDir(), randomStr); File dataTestDir = new File(newDataTestDir.toString()); if (deleteOnExit()) dataTestDir.deleteOnExit(); } else { Path base = getBaseTestDirOnTestFS(); newDataTestDir = new Path(base, randomStr); if (deleteOnExit()) fs.deleteOnExit(newDataTestDir); } return newDataTestDir; }
LocalFileSystem localFileSystem = new LocalFileSystem(); Path outputDir = new Path(outputPath).makeQualified(localFileSystem.getUri(), localFileSystem.getWorkingDirectory()); byte[][] families = { FAMILY_A, FAMILY_B };
private static Path downloadResource(String destName, URI uri, Configuration conf, String destDir) throws IOException { FileSystem fs = FileSystem.get(uri, conf); Path cwd = new Path(destDir); Path dFile = new Path(cwd, destName); Path srcPath = new Path(uri); fs.copyToLocalFile(srcPath, dFile); return dFile.makeQualified(FileSystem.getLocal(conf).getUri(), cwd); } }
+ "the input trace source is a stream instead of file."); emulateDistributedCache = generateDistCacheData = false; } else if (FileSystem.getLocal(conf).getUri().getScheme().equals(
if (blockListFS.getUri().equals(FileSystem.getLocal(getConf()).getUri()) || !blockListFS.exists(blockPath)) { throw new IllegalArgumentException("block list path must already exist on remote fs!");
private void checkLocalFunctionResources(Hive db, List<ResourceUri> resources) throws HiveException { // If this is a non-local warehouse, then adding resources from the local filesystem // may mean that other clients will not be able to access the resources. // So disallow resources from local filesystem in this case. if (resources != null && resources.size() > 0) { try { String localFsScheme = FileSystem.getLocal(db.getConf()).getUri().getScheme(); String configuredFsScheme = FileSystem.get(db.getConf()).getUri().getScheme(); if (configuredFsScheme.equals(localFsScheme)) { // Configured warehouse FS is local, don't need to bother checking. return; } for (ResourceUri res : resources) { String resUri = res.getUri(); if (!SessionState.canDownloadResource(resUri)) { throw new HiveException("Hive warehouse is non-local, but " + res.getUri() + " specifies file on local filesystem. " + "Resources on non-local warehouse should specify a non-local scheme/path"); } } } catch (HiveException e) { throw e; } catch (Exception e) { LOG.error(e); throw new HiveException(e); } } }
/** * Sets up a new path in test filesystem to be used by tests. */ private Path getNewDataTestDirOnTestFS() throws IOException { //The file system can be either local, mini dfs, or if the configuration //is supplied externally, it can be an external cluster FS. If it is a local //file system, the tests should use getBaseTestDir, otherwise, we can use //the working directory, and create a unique sub dir there FileSystem fs = getTestFileSystem(); Path newDataTestDir; String randomStr = getRandomUUID().toString(); if (fs.getUri().getScheme().equals(FileSystem.getLocal(conf).getUri().getScheme())) { newDataTestDir = new Path(getDataTestDir(), randomStr); File dataTestDir = new File(newDataTestDir.toString()); if (deleteOnExit()) dataTestDir.deleteOnExit(); } else { Path base = getBaseTestDirOnTestFS(); newDataTestDir = new Path(base, randomStr); if (deleteOnExit()) fs.deleteOnExit(newDataTestDir); } return newDataTestDir; }
@Test public void testInitLocalFsSetsConfs() throws Exception { LocalFileSystem lfs = new LocalFileSystem(); checkFsConf(lfs, null, 2); lfs.initialize(lfs.getUri(), conf); checkFsConf(lfs, conf, 2); }
@Test public void testInitLocalFsSetsConfs() throws Exception { LocalFileSystem lfs = new LocalFileSystem(); checkFsConf(lfs, null, 2); lfs.initialize(lfs.getUri(), conf); checkFsConf(lfs, conf, 2); }
LocalFileSystem localFileSystem = new LocalFileSystem(); Path outputDir = new Path(outputPath).makeQualified(localFileSystem.getUri(), localFileSystem.getWorkingDirectory()); byte[][] families = { FAMILY_A, FAMILY_B };
LocalFileSystem localFileSystem = new LocalFileSystem(); Path outputDir = new Path(outputPath).makeQualified(localFileSystem.getUri(), localFileSystem.getWorkingDirectory()); byte[][] families = { FAMILY_A, FAMILY_B };