/** * Returns the configured FileSystem implementation. * @param conf the configuration to use */ public static FileSystem get(Configuration conf) throws IOException { return get(getDefaultUri(conf), conf); }
private static URI resolveUri(URI uri, Configuration configuration, FileSystem fileSystem) throws IOException { if (uri != null) { return uri; } if (fileSystem != null) { return fileSystem.getUri(); } if (configuration != null) { return FileSystem.getDefaultUri(configuration); } throw new IOException("FileSystem URI could not be determined from available inputs."); }
/** * Takes Fs URI using logic similar to that used in FileSystem#get(1,2,3). * @param uri0 The uri. * @param cfg The cfg. * @return Correct URI. */ private static URI fixUri(URI uri0, Configuration cfg) { if (uri0 == null) return FileSystem.getDefaultUri(cfg); String scheme = uri0.getScheme(); String authority = uri0.getAuthority(); if (authority == null) { URI dfltUri = FileSystem.getDefaultUri(cfg); if (scheme == null || (scheme.equals(dfltUri.getScheme()) && dfltUri.getAuthority() != null)) return dfltUri; } return uri0; }
/** * Returns a unique configured FileSystem implementation for the default * filesystem of the supplied configuration. * This always returns a new FileSystem object. * @param conf the configuration to use * @return the new FS instance * @throws IOException FS creation or initialization failure. */ public static FileSystem newInstance(Configuration conf) throws IOException { return newInstance(getDefaultUri(conf), conf); }
/** * Equivalent to {@link FileSystem#get(Configuration)}, but uses the input {@link SharedResourcesBroker} to configure * add-ons to the {@link FileSystem} (e.g. throttling, instrumentation). */ public static <S extends ScopeType<S>> FileSystem get(Configuration configuration, SharedResourcesBroker<S> broker) throws IOException { return get(FileSystem.getDefaultUri(configuration), configuration, broker); }
private URI resolveURI(URI uri, Configuration configuration) { String scheme = uri.getScheme(); String authority = uri.getAuthority(); if (scheme == null && authority == null) { // use default FS return FileSystem.getDefaultUri(configuration); } if (scheme != null && authority == null) { // no authority URI defaultUri = FileSystem.getDefaultUri(configuration); if (scheme.equals(defaultUri.getScheme()) // if scheme matches default && defaultUri.getAuthority() != null) { // & default has authority return defaultUri; // return default } } try { return new URI(scheme, Strings.nullToEmpty(authority), "/", null, null); } catch (URISyntaxException use) { // This should never happen throw new RuntimeException(use); } }
/** * Gets a {@link FileSystem} that can perform any operations allowed by the specified userNameToProxyAs. * * @param userNameToProxyAs The name of the user the super user should proxy as * @param properties {@link java.util.Properties} containing initialization properties. * @param conf The {@link Configuration} for the {@link FileSystem} that should be created. * @return a {@link FileSystem} that can execute commands on behalf of the specified userNameToProxyAs * @throws IOException * @deprecated use {@link #fromProperties} */ @Deprecated public static FileSystem getProxiedFileSystem(@NonNull final String userNameToProxyAs, Properties properties, Configuration conf) throws IOException { return getProxiedFileSystem(userNameToProxyAs, properties, FileSystem.getDefaultUri(conf), conf); }
@Override public FileSystem create(URI fsUri) throws IOException { this.hadoopConfig = getHadoopConfiguration(); final String scheme = fsUri.getScheme(); final String authority = fsUri.getAuthority(); if (scheme == null && authority == null) { fsUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConfig); } else if (scheme != null && authority == null) { URI defaultUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConfig); if (scheme.equals(defaultUri.getScheme()) && defaultUri.getAuthority() != null) { fsUri = defaultUri; } } final AliyunOSSFileSystem fs = new AliyunOSSFileSystem(); fs.initialize(fsUri, hadoopConfig); return new HadoopFileSystem(fs); }
public static boolean isDefaultFs(DistributedFileSystem fs) { URI uri = fs.getUri(); String scheme = uri.getScheme(); if (scheme == null) return true; // Assume that relative URI resolves to default FS. URI defaultUri = FileSystem.getDefaultUri(fs.getConf()); if (!defaultUri.getScheme().equalsIgnoreCase(scheme)) return false; // Mismatch. String defaultAuthority = defaultUri.getAuthority(), authority = uri.getAuthority(); if (authority == null) return true; // Schemes match, no authority - assume default. if (defaultAuthority == null) return false; // TODO: What does this even mean? if (!defaultUri.getHost().equalsIgnoreCase(uri.getHost())) return false; // Mismatch. int defaultPort = defaultUri.getPort(), port = uri.getPort(); if (port == -1) return true; // No port, assume default. // Note - this makes assumptions that are DFS-specific; DFS::getDefaultPort is not visible. return (defaultPort == -1) ? (port == NameNode.DEFAULT_PORT) : (port == defaultPort); } }
/** * Lazy loading of YARN configuration since it takes a long time to load. * (YARN provides no caching, sadly.) */ private void loadYarnConfig() { if (yarnConf == null) { yarnConf = new YarnConfiguration(); // On some distributions, lack of proper configuration causes // DFS to default to the local file system. So, a local file // system generally means that the config is wrong, or running // the wrong build of Drill for the user's environment. URI fsUri = FileSystem.getDefaultUri( yarnConf ); if(fsUri.toString().startsWith("file:/")) { System.err.println("Warning: Default DFS URI is for a local file system: " + fsUri); } } }
/** * Returns a brand new instance of the FileSystem. It does not use * the FileSystem.Cache. In newer versions of HDFS, we can directly * invoke FileSystem.newInstance(Configuration). * * @param conf Configuration * @return A new instance of the filesystem */ private static FileSystem newInstanceFileSystem(Configuration conf) throws IOException { URI uri = FileSystem.getDefaultUri(conf); FileSystem fs = null; Class<?> clazz = conf.getClass("fs." + uri.getScheme() + ".impl", null); if (clazz != null) { // This will be true for Hadoop 1.0, or 0.20. fs = (FileSystem) org.apache.hadoop.util.ReflectionUtils.newInstance(clazz, conf); fs.initialize(uri, conf); } else { // For Hadoop 2.0, we have to go through FileSystem for the filesystem // implementation to be loaded by the service loader in case it has not // been loaded yet. Configuration clone = new Configuration(conf); clone.setBoolean("fs." + uri.getScheme() + ".impl.disable.cache", true); fs = FileSystem.get(uri, clone); } if (fs == null) { throw new IOException("No FileSystem for scheme: " + uri.getScheme()); } return fs; }
/** * Initialize a FileSystem. * * Called after the new FileSystem instance is constructed, and before it * is ready for use. * * FileSystem implementations overriding this method MUST forward it to * their superclass, though the order in which it is done, and whether * to alter the configuration before the invocation are options of the * subclass. * @param name a URI whose authority section names the host, port, etc. * for this FileSystem * @param conf the configuration * @throws IOException on any failure to initialize this instance. * @throws IllegalArgumentException if the URI is considered invalid. */ public void initialize(URI name, Configuration conf) throws IOException { final String scheme; if (name.getScheme() == null || name.getScheme().isEmpty()) { scheme = getDefaultUri(conf).getScheme(); } else { scheme = name.getScheme(); } statistics = getStatistics(scheme, getClass()); resolveSymlinks = conf.getBoolean( CommonConfigurationKeysPublic.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_KEY, CommonConfigurationKeysPublic.FS_CLIENT_RESOLVE_REMOTE_SYMLINKS_DEFAULT); }
: FileSystem.getDefaultUri(configuration);
protected void checkHdfsUriForTimeout(Configuration config) throws IOException { URI hdfsUri = FileSystem.getDefaultUri(config); String address = hdfsUri.getAuthority(); int port = hdfsUri.getPort(); if (address == null || address.isEmpty() || port < 0) { return; } InetSocketAddress namenode = NetUtils.createSocketAddr(address, port); SocketFactory socketFactory = NetUtils.getDefaultSocketFactory(config); Socket socket = null; try { socket = socketFactory.createSocket(); NetUtils.connect(socket, namenode, 1000); // 1 second timeout } finally { IOUtils.closeQuietly(socket); } }
fsUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConfig); URI defaultUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConfig); if (scheme.equals(defaultUri.getScheme()) && defaultUri.getAuthority() != null) { fsUri = defaultUri;
/** * Returns the FileSystem for this URI's scheme and authority. * The entire URI is passed to the FileSystem instance's initialize method. * This always returns a new FileSystem object. * @param uri FS URI * @param config configuration to use * @return the new FS instance * @throws IOException FS creation or initialization failure. */ public static FileSystem newInstance(URI uri, Configuration config) throws IOException { String scheme = uri.getScheme(); String authority = uri.getAuthority(); if (scheme == null) { // no scheme: use default FS return newInstance(config); } if (authority == null) { // no authority URI defaultUri = getDefaultUri(config); if (scheme.equals(defaultUri.getScheme()) // if scheme matches default && defaultUri.getAuthority() != null) { // & default has authority return newInstance(defaultUri, config); // return default } } return CACHE.getUnique(uri, config); }
fsUri = FileSystem.getDefaultUri(finalConf); URI defaultUri = FileSystem.getDefaultUri(finalConf); if (scheme.equals(defaultUri.getScheme()) && defaultUri.getAuthority() != null) { fsUri = defaultUri;
final URI fsUri = org.apache.hadoop.fs.FileSystem.getDefaultUri(hadoopConf); if (fsUri.getScheme() == null || !"hdfs".equals(fsUri.getScheme().toLowerCase())) { throw new IOException("Invalid file system found for YarnHighAvailabilityServices: " +
uri = FileSystem.getDefaultUri(cfg);
@Test public void testFsUriSetProperly() throws Exception { HMaster master = UTIL.getMiniHBaseCluster().getMaster(); MasterFileSystem fs = master.getMasterFileSystem(); Path masterRoot = FSUtils.getRootDir(fs.getConfiguration()); Path rootDir = FSUtils.getRootDir(fs.getFileSystem().getConf()); // make sure the fs and the found root dir have the same scheme LOG.debug("from fs uri:" + FileSystem.getDefaultUri(fs.getFileSystem().getConf())); LOG.debug("from configuration uri:" + FileSystem.getDefaultUri(fs.getConfiguration())); // make sure the set uri matches by forcing it. assertEquals(masterRoot, rootDir); } }