/** * @param aConf - from which the FileContext is configured * @return a FileContext for the local file system using the specified config. * * @throws UnsupportedFileSystemException If default file system in the config * is not supported * */ public static FileContext getLocalFSFileContext(final Configuration aConf) throws UnsupportedFileSystemException { return getFileContext(FsConstants.LOCAL_FS_URI, aConf); }
/** * @return a FileContext for the local file system using the default config. * @throws UnsupportedFileSystemException If the file system for * {@link FsConstants#LOCAL_FS_URI} is not supported. */ public static FileContext getLocalFSFileContext() throws UnsupportedFileSystemException { return getFileContext(FsConstants.LOCAL_FS_URI); }
/** * Create a FileContext for specified file system using the default config. * * @param defaultFS * @return a FileContext with the specified AbstractFileSystem * as the default FS. */ protected static FileContext getFileContext( final AbstractFileSystem defaultFS) { return getFileContext(defaultFS, new Configuration()); }
/** * Create a FileContext using the default config read from the * $HADOOP_CONFIG/core.xml, Unspecified key-values for config are defaulted * from core-defaults.xml in the release jar. * * @throws UnsupportedFileSystemException If the file system from the default * configuration is not supported */ public static FileContext getFileContext() throws UnsupportedFileSystemException { return getFileContext(new Configuration()); }
/** * Create a FileContext for specified URI using the default config. * * @param defaultFsUri * @return a FileContext with the specified URI as the default FS. * * @throws UnsupportedFileSystemException If the file system for * <code>defaultFsUri</code> is not supported */ public static FileContext getFileContext(final URI defaultFsUri) throws UnsupportedFileSystemException { return getFileContext(defaultFsUri, new Configuration()); }
/** * Create a FileContext for specified default URI using the specified config. * * @param defaultFsUri * @param aConf * @return new FileContext for specified uri * @throws UnsupportedFileSystemException If the file system with specified is * not supported * @throws RuntimeException If the file system specified is supported but * could not be instantiated, or if login fails. */ public static FileContext getFileContext(final URI defaultFsUri, final Configuration aConf) throws UnsupportedFileSystemException { UserGroupInformation currentUser = null; AbstractFileSystem defaultAfs = null; if (defaultFsUri.getScheme() == null) { return getFileContext(aConf); } try { currentUser = UserGroupInformation.getCurrentUser(); defaultAfs = getAbstractFileSystem(currentUser, defaultFsUri, aConf); } catch (UnsupportedFileSystemException ex) { throw ex; } catch (IOException ex) { LOG.error(ex.toString()); throw new RuntimeException(ex); } return getFileContext(defaultAfs, aConf); }
@Override public void commit() throws IOException { checkIsNotTmp(); // FileContext supports atomic rename, whereas FileSystem doesn't FileContext fc = FileContext.getFileContext(_hadoopConf); Path dest = new Path(_path.getParent(), BLOBSTORE_DATA_FILE); if (_mustBeNew) { fc.rename(_path, dest); } else { fc.rename(_path, dest, Options.Rename.OVERWRITE); } // Note, we could add support for setting the replication factor }
/** * Create a FileContext using the passed config. Generally it is better to use * {@link #getFileContext(URI, Configuration)} instead of this one. * * * @param aConf * @return new FileContext * @throws UnsupportedFileSystemException If file system in the config * is not supported */ public static FileContext getFileContext(final Configuration aConf) throws UnsupportedFileSystemException { final URI defaultFsUri = URI.create(aConf.get(FS_DEFAULT_NAME_KEY, FS_DEFAULT_NAME_DEFAULT)); if ( defaultFsUri.getScheme() != null && !defaultFsUri.getScheme().trim().isEmpty()) { return getFileContext(defaultFsUri, aConf); } throw new UnsupportedFileSystemException(String.format( "%s: URI configured via %s carries no scheme", defaultFsUri, FS_DEFAULT_NAME_KEY)); }
public AvroFileInputStream(FileStatus status) throws IOException { pos = 0; buffer = new byte[0]; GenericDatumReader<Object> reader = new GenericDatumReader<Object>(); FileContext fc = FileContext.getFileContext(new Configuration()); fileReader = DataFileReader.openReader(new AvroFSInput(fc, status.getPath()),reader); Schema schema = fileReader.getSchema(); writer = new GenericDatumWriter<Object>(schema); output = new ByteArrayOutputStream(); encoder = EncoderFactory.get().jsonEncoder(schema, output); }
/** * Returns all jars that are in the directory. It is useful in expanding a * wildcard path to return all jars from the directory to use in a classpath. * * @param path the path to the directory. The path may include the wildcard. * @return the list of jars as URLs, or an empty list if there are no jars, or * the directory does not exist */ public static List<Path> getJarsInDirectory(String path, boolean useLocal) { List<Path> paths = new ArrayList<>(); try { // add the wildcard if it is not provided if (!path.endsWith("*")) { path += File.separator + "*"; } Path globPath = new Path(path).suffix("{.jar,.JAR}"); FileContext context = useLocal ? FileContext.getLocalFSFileContext() : FileContext.getFileContext(globPath.toUri()); FileStatus[] files = context.util().globStatus(globPath); if (files != null) { for (FileStatus file: files) { paths.add(file.getPath()); } } } catch (IOException ignore) {} // return the empty list return paths; }
try { destUrl = ConverterUtils.getYarnUrlFromPath( FileContext.getFileContext().makeQualified(dfsPath)); } catch (UnsupportedFileSystemException e) { throw new DfsFacadeException(
CompressionType compressionType, CompressionCodec codec, Metadata metadata) throws IOException { return createWriter(FileContext.getFileContext(fs.getUri(), conf), conf, name, keyClass, valClass, compressionType, codec, metadata, EnumSet.of(CreateFlag.CREATE,CreateFlag.OVERWRITE),
URI uri = URI.create(uriStr); this.fs = FileSystem.get(uri, conf); this.fileContext = FileContext.getFileContext(uri, conf);
this.fileContext = FileContext.getFileContext(conf);
/** * @return a FileContext for the local file system using the default config. * @throws UnsupportedFileSystemException If the file system for * {@link FsConstants#LOCAL_FS_URI} is not supported. */ public static FileContext getLocalFSFileContext() throws UnsupportedFileSystemException { return getFileContext(FsConstants.LOCAL_FS_URI); }
/** * Create a FileContext for specified file system using the default config. * * @param defaultFS * @return a FileContext with the specified AbstractFileSystem * as the default FS. */ protected static FileContext getFileContext( final AbstractFileSystem defaultFS) { return getFileContext(defaultFS, new Configuration()); }
@Override public FSDataOutputStream run() throws Exception { fc = FileContext.getFileContext(remoteAppLogFile.toUri(), conf); fc.setUMask(APP_LOG_FILE_UMASK); return fc.create( remoteAppLogFile, EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), new Options.CreateOpts[] {}); } });
@Override public void open(String pathStr, String singleFileOffset) { LOG.info(String.format("%s: Open file [%s] with file offset [%s] for read", systemStreamPartition, pathStr, singleFileOffset)); Path path = new Path(pathStr); try { AvroFSInput input = new AvroFSInput(FileContext.getFileContext(path.toUri()), path); fileReader = new DataFileReader<>(input, new GenericDatumReader<>()); seek(singleFileOffset); } catch (IOException e) { throw new SamzaException(e); } }
@Override @Before public void setUp() throws Exception { Configuration S3Conf = new Configuration(); Configuration localConf = new Configuration(); S3Conf.set(FS_DEFAULT_NAME_DEFAULT, S3Conf.get("test.fs.s3.name")); fc1 = FileContext.getFileContext(S3Conf); fc2 = FileContext.getFileContext(localConf); }
@Override public Configuration loadConfFile() throws IOException { FileContext fc = FileContext.getFileContext(configFile.toUri(), conf); Configuration jobConf = new Configuration(false); jobConf.addResource(fc.open(configFile), configFile.toString()); return jobConf; }