How to use

UnderFileSystemUtils

in

alluxio.util

 @Test
 public void getBucketName() throws Exception {
  Assert.assertEquals("s3-bucket-name",
    UnderFileSystemUtils.getBucketName(new AlluxioURI("s3://s3-bucket-name/")));
  Assert.assertEquals("s3a_bucket_name",
    UnderFileSystemUtils.getBucketName(new AlluxioURI("s3a://s3a_bucket_name/")));
  Assert.assertEquals("a.b.c",
    UnderFileSystemUtils.getBucketName(new AlluxioURI("gs://a.b.c/folder/sub-folder/")));
  Assert.assertEquals("container&",
    UnderFileSystemUtils.getBucketName(new AlluxioURI("swift://container&/folder/file")));
  Assert.assertEquals("oss",
    UnderFileSystemUtils.getBucketName(new AlluxioURI("oss://oss/folder/.file")));
 }
}

@Override
public UfsFileStatus getFileStatus(String path) throws IOException {
 Path tPath = new Path(path);
 FileSystem hdfs = getFs();
 FileStatus fs = hdfs.getFileStatus(tPath);
 String contentHash =
   UnderFileSystemUtils.approximateContentHash(fs.getLen(), fs.getModificationTime());
 return new UfsFileStatus(path, contentHash, fs.getLen(),
   fs.getModificationTime(), fs.getOwner(), fs.getGroup(), fs.getPermission().toShort());
}

 /**
  * Updates the checkpoint to the specified URI.
  *
  * @param location the location of the new checkpoint
  */
 public void update(URI location) {
  try {
   if (mUfs.isFile(mCheckpoint.toString())) {
    UnderFileSystemUtils.deleteFileIfExists(mUfs, mTempBackupCheckpoint.toString());
    UnderFileSystemUtils.deleteFileIfExists(mUfs, mBackupCheckpoint.toString());
    // Rename in two steps so that we never have identical mCheckpointPath and
    // mBackupCheckpointPath. This is a concern since UFS may implement rename as copy + delete.
    mUfs.renameFile(mCheckpoint.toString(), mTempBackupCheckpoint.toString());
    mUfs.renameFile(mTempBackupCheckpoint.toString(), mBackupCheckpoint.toString());
    LOG.info("Backed up the checkpoint file to {}", mBackupCheckpoint.toString());
   }
   mUfs.renameFile(location.getPath(), mCheckpoint.toString());
   LOG.info("Renamed the checkpoint file from {} to {}", location,
     mCheckpoint.toString());

   // The checkpoint already reflects the information in the completed logs.
   mWriter.deleteCompletedLogs();
   UnderFileSystemUtils.deleteFileIfExists(mUfs, mBackupCheckpoint.toString());
  } catch (IOException e) {
   throw new RuntimeException(e);
  }
 }
}

/**
 * Formats the journal.
 */
public void format() throws IOException {
 URI location = getLocation();
 LOG.info("Formatting {}", location);
 if (mUfs.isDirectory(location.toString())) {
  for (UfsStatus status : mUfs.listStatus(location.toString())) {
   String childPath = URIUtils.appendPathOrDie(location, status.getName()).toString();
   if (status.isDirectory()
     && !mUfs.deleteDirectory(childPath, DeleteOptions.defaults().setRecursive(true))
     || status.isFile() && !mUfs.deleteFile(childPath)) {
    throw new IOException(String.format("Failed to delete %s", childPath));
   }
  }
 } else if (!mUfs.mkdirs(location.toString())) {
  throw new IOException(String.format("Failed to create %s", location));
 }
 // Create a breadcrumb that indicates that the journal folder has been formatted.
 UnderFileSystemUtils.touch(mUfs, URIUtils.appendPathOrDie(location,
   ServerConfiguration.get(PropertyKey.MASTER_FORMAT_FILE_PREFIX) + System.currentTimeMillis())
   .toString());
}

@Override
public TaskResult validate(Map<String, String> optionsMap) {
 if (!UnderFileSystemUtils.isHdfs(mUfs)) {

UnderFileSystemUtils.touch(ufs, URIUtils.appendPath(mLocation,
  ServerConfiguration.get(PropertyKey.MASTER_FORMAT_FILE_PREFIX)
    + System.currentTimeMillis())

@Test
public void createOpenEmpty() throws IOException {
 String testFile = PathUtils.concatPath(mUnderfsAddress, "createOpenEmpty");
 createEmptyFile(testFile);
 byte[] buf = new byte[0];
 int bytesRead = mUfs.open(testFile).read(buf);
 // TODO(adit): Consider making the return value uniform across UFSs
 if (UnderFileSystemUtils.isHdfs(mUfs)) {
  assertEquals(-1, bytesRead);
 } else {
  assertEquals(0, bytesRead);
 }
}

/**
 * Constructs a new instance of {@link OSSUnderFileSystem}.
 *
 * @param uri the {@link AlluxioURI} for this UFS
 * @param conf the configuration for this UFS
 * @param alluxioConf Alluxio configuration
 * @return the created {@link OSSUnderFileSystem} instance
 */
public static OSSUnderFileSystem createInstance(AlluxioURI uri,
  UnderFileSystemConfiguration conf, AlluxioConfiguration alluxioConf) throws Exception {
 String bucketName = UnderFileSystemUtils.getBucketName(uri);
 Preconditions.checkArgument(conf.isSet(PropertyKey.OSS_ACCESS_KEY),
   "Property %s is required to connect to OSS", PropertyKey.OSS_ACCESS_KEY);
 Preconditions.checkArgument(conf.isSet(PropertyKey.OSS_SECRET_KEY),
   "Property %s is required to connect to OSS", PropertyKey.OSS_SECRET_KEY);
 Preconditions.checkArgument(conf.isSet(PropertyKey.OSS_ENDPOINT_KEY),
   "Property %s is required to connect to OSS", PropertyKey.OSS_ENDPOINT_KEY);
 String accessId = conf.get(PropertyKey.OSS_ACCESS_KEY);
 String accessKey = conf.get(PropertyKey.OSS_SECRET_KEY);
 String endPoint = conf.get(PropertyKey.OSS_ENDPOINT_KEY);
 ClientConfiguration ossClientConf = initializeOSSClientConfig(alluxioConf);
 OSSClient ossClient = new OSSClient(endPoint, accessId, accessKey, ossClientConf);
 return new OSSUnderFileSystem(uri, ossClient, bucketName, conf, alluxioConf);
}

@Override
public synchronized JournalOutputStream getCheckpointOutputStream(long latestSequenceNumber)
  throws IOException {
 if (mCheckpointOutputStream == null) {
  mCheckpointManager.recover();
  LOG.info("Creating tmp checkpoint file: {}", mTempCheckpoint);
  if (!mUfs.isDirectory(mJournal.getLocation().toString())) {
   LOG.info("Creating journal folder: {}", mJournal.getLocation());
   mUfs.mkdirs(mJournal.getLocation().toString());
  }
  mNextEntrySequenceNumber = latestSequenceNumber + 1;
  LOG.info("Latest journal sequence number: {} Next journal sequence number: {}",
    latestSequenceNumber, mNextEntrySequenceNumber);
  UnderFileSystemUtils.deleteFileIfExists(mUfs, mTempCheckpoint.toString());
  mCheckpointOutputStream = new CheckpointOutputStream(
    new DataOutputStream(mUfs.create(mTempCheckpoint.toString())));
 }
 return mCheckpointOutputStream;
}

@Override
public UfsFileStatus getFileStatus(String path) throws IOException {
 String tpath = stripPath(path);
 File file = new File(tpath);
 PosixFileAttributes attr =
   Files.readAttributes(Paths.get(file.getPath()), PosixFileAttributes.class);
 String contentHash =
   UnderFileSystemUtils.approximateContentHash(file.length(), file.lastModified());
 return new UfsFileStatus(path, contentHash, file.length(),
   file.lastModified(), attr.owner().getName(), attr.group().getName(),
   FileUtils.translatePosixPermissionToMode(attr.permissions()));
}

UnderFileSystem ufs = UnderFileSystem.Factory.create(job.getTempUfsPath(),
  ServerConfiguration.global());
UnderFileSystemUtils.touch(ufs, job.getTempUfsPath());

@Override
public TaskResult validate(Map<String, String> optionsMap) {
 if (!UnderFileSystemUtils.isHdfs(mUfs)) {

protected static KodoUnderFileSystem creatInstance(AlluxioURI uri,
  UnderFileSystemConfiguration conf, AlluxioConfiguration alluxioConf) {
 String bucketName = UnderFileSystemUtils.getBucketName(uri);
 Preconditions.checkArgument(conf.isSet(PropertyKey.KODO_ACCESS_KEY),
   "Property %s is required to connect to Kodo", PropertyKey.KODO_ACCESS_KEY);
 Preconditions.checkArgument(conf.isSet(PropertyKey.KODO_SECRET_KEY),
   "Property %s is required to connect to Kodo", PropertyKey.KODO_SECRET_KEY);
 Preconditions.checkArgument(conf.isSet(PropertyKey.KODO_DOWNLOAD_HOST),
   "Property %s is required to connect to Kodo", PropertyKey.KODO_DOWNLOAD_HOST);
 Preconditions.checkArgument(conf.isSet(PropertyKey.KODO_ENDPOINT),
   "Property %s is required to connect to Kodo", PropertyKey.KODO_ENDPOINT);
 String accessKey = conf.get(PropertyKey.KODO_ACCESS_KEY);
 String secretKey = conf.get(PropertyKey.KODO_SECRET_KEY);
 String endPoint = conf.get(PropertyKey.KODO_ENDPOINT);
 String souceHost = conf.get(PropertyKey.KODO_DOWNLOAD_HOST);
 Auth auth = Auth.create(accessKey, secretKey);
 Configuration configuration = new Configuration();
 OkHttpClient.Builder okHttpBuilder = initializeKodoClientConfig(conf);
 OkHttpClient okHttpClient = okHttpBuilder.build();
 KodoClient kodoClient =
   new KodoClient(auth, bucketName, souceHost, endPoint, configuration, okHttpClient);
 return new KodoUnderFileSystem(uri, kodoClient, conf, alluxioConf);
}

UnderFileSystemUtils.deleteFileIfExists(mUfs, mCheckpoint.toString());
mUfs.renameFile(mTempBackupCheckpoint.toString(), mCheckpoint.toString());

@Override
public UfsStatus[] listStatus(String path) throws IOException {
 path = stripPath(path);
 File file = new File(path);
 File[] files = file.listFiles();
 if (files != null) {
  UfsStatus[] rtn = new UfsStatus[files.length];
  int i = 0;
  for (File f : files) {
   // TODO(adit): do we need extra call for attributes?
   PosixFileAttributes attr =
     Files.readAttributes(Paths.get(f.getPath()), PosixFileAttributes.class);
   short mode = FileUtils.translatePosixPermissionToMode(attr.permissions());
   UfsStatus retStatus;
   if (f.isDirectory()) {
    retStatus = new UfsDirectoryStatus(f.getName(), attr.owner().getName(),
      attr.group().getName(), mode, f.lastModified());
   } else {
    String contentHash =
      UnderFileSystemUtils.approximateContentHash(f.length(), f.lastModified());
    retStatus = new UfsFileStatus(f.getName(), contentHash, f.length(), f.lastModified(),
      attr.owner().getName(), attr.group().getName(), mode);
   }
   rtn[i++] = retStatus;
  }
  return rtn;
 } else {
  return null;
 }
}

UnderFileSystem ufs = UnderFileSystem.Factory.create(job.getTempUfsPath(),
  ServerConfiguration.global());
UnderFileSystemUtils.touch(ufs, job.getTempUfsPath());

/**
 * Constructs a new instance of {@link COSUnderFileSystem}.
 *
 * @param uri the {@link AlluxioURI} for this UFS
 * @param conf the configuration for this UFS
 * @param alluxioConf Alluxio configuration
 * @return the created {@link COSUnderFileSystem} instance
 */
public static COSUnderFileSystem createInstance(AlluxioURI uri, UnderFileSystemConfiguration conf,
  AlluxioConfiguration alluxioConf)
  throws Exception {
 String bucketName = UnderFileSystemUtils.getBucketName(uri);
 Preconditions.checkArgument(conf.isSet(PropertyKey.COS_ACCESS_KEY),
   "Property %s is required to connect to COS", PropertyKey.COS_ACCESS_KEY);
 Preconditions.checkArgument(conf.isSet(PropertyKey.COS_SECRET_KEY),
   "Property %s is required to connect to COS", PropertyKey.COS_SECRET_KEY);
 Preconditions.checkArgument(conf.isSet(PropertyKey.COS_REGION),
   "Property %s is required to connect to COS", PropertyKey.COS_REGION);
 Preconditions.checkArgument(conf.isSet(PropertyKey.COS_APP_ID),
   "Property %s is required to connect to COS", PropertyKey.COS_APP_ID);
 String accessKey = conf.get(PropertyKey.COS_ACCESS_KEY);
 String secretKey = conf.get(PropertyKey.COS_SECRET_KEY);
 String regionName = conf.get(PropertyKey.COS_REGION);
 String appId = conf.get(PropertyKey.COS_APP_ID);
 COSCredentials cred = new BasicCOSCredentials(accessKey, secretKey);
 ClientConfig clientConfig = createCOSClientConfig(regionName, conf);
 COSClient client = new COSClient(cred, clientConfig);
 return new COSUnderFileSystem(uri, client, bucketName, appId, conf, alluxioConf);
}

String ufsBlockPath = alluxio.worker.BlockUtils.getUfsBlockPath(ufsClient, blockId);
try (CloseableResource<UnderFileSystem> ufsResource = ufsClient.acquireUfsResource()) {
 alluxio.util.UnderFileSystemUtils.deleteFileIfExists(ufsResource.get(), ufsBlockPath);
} catch (Exception e) {
 LOG.warn("Failed to clean up staging UFS block file {}", ufsBlockPath, e.getMessage());

@Override
public UfsStatus getStatus(String path) throws IOException {
 String tpath = stripPath(path);
 File file = new File(tpath);
 PosixFileAttributes attr =
   Files.readAttributes(Paths.get(file.getPath()), PosixFileAttributes.class);
 if (file.isFile()) {
  // Return file status.
  String contentHash =
    UnderFileSystemUtils.approximateContentHash(file.length(), file.lastModified());
  return new UfsFileStatus(path, contentHash, file.length(), file.lastModified(),
    attr.owner().getName(), attr.group().getName(),
    FileUtils.translatePosixPermissionToMode(attr.permissions()));
 }
 // Return directory status.
 return new UfsDirectoryStatus(path, attr.owner().getName(), attr.group().getName(),
   FileUtils.translatePosixPermissionToMode(attr.permissions()), file.lastModified());
}

UnderFileSystemUtils.touch(ufs, URIUtils.appendPath(mLocation,
  Configuration.get(PropertyKey.MASTER_FORMAT_FILE_PREFIX) + System.currentTimeMillis())
  .toString());