@Test public void getBucketName() throws Exception { Assert.assertEquals("s3-bucket-name", UnderFileSystemUtils.getBucketName(new AlluxioURI("s3://s3-bucket-name/"))); Assert.assertEquals("s3a_bucket_name", UnderFileSystemUtils.getBucketName(new AlluxioURI("s3a://s3a_bucket_name/"))); Assert.assertEquals("a.b.c", UnderFileSystemUtils.getBucketName(new AlluxioURI("gs://a.b.c/folder/sub-folder/"))); Assert.assertEquals("container&", UnderFileSystemUtils.getBucketName(new AlluxioURI("swift://container&/folder/file"))); Assert.assertEquals("oss", UnderFileSystemUtils.getBucketName(new AlluxioURI("oss://oss/folder/.file"))); } }
@Override public UfsFileStatus getFileStatus(String path) throws IOException { Path tPath = new Path(path); FileSystem hdfs = getFs(); FileStatus fs = hdfs.getFileStatus(tPath); String contentHash = UnderFileSystemUtils.approximateContentHash(fs.getLen(), fs.getModificationTime()); return new UfsFileStatus(path, contentHash, fs.getLen(), fs.getModificationTime(), fs.getOwner(), fs.getGroup(), fs.getPermission().toShort()); }
/** * Updates the checkpoint to the specified URI. * * @param location the location of the new checkpoint */ public void update(URI location) { try { if (mUfs.isFile(mCheckpoint.toString())) { UnderFileSystemUtils.deleteFileIfExists(mUfs, mTempBackupCheckpoint.toString()); UnderFileSystemUtils.deleteFileIfExists(mUfs, mBackupCheckpoint.toString()); // Rename in two steps so that we never have identical mCheckpointPath and // mBackupCheckpointPath. This is a concern since UFS may implement rename as copy + delete. mUfs.renameFile(mCheckpoint.toString(), mTempBackupCheckpoint.toString()); mUfs.renameFile(mTempBackupCheckpoint.toString(), mBackupCheckpoint.toString()); LOG.info("Backed up the checkpoint file to {}", mBackupCheckpoint.toString()); } mUfs.renameFile(location.getPath(), mCheckpoint.toString()); LOG.info("Renamed the checkpoint file from {} to {}", location, mCheckpoint.toString()); // The checkpoint already reflects the information in the completed logs. mWriter.deleteCompletedLogs(); UnderFileSystemUtils.deleteFileIfExists(mUfs, mBackupCheckpoint.toString()); } catch (IOException e) { throw new RuntimeException(e); } } }
/** * Formats the journal. */ public void format() throws IOException { URI location = getLocation(); LOG.info("Formatting {}", location); if (mUfs.isDirectory(location.toString())) { for (UfsStatus status : mUfs.listStatus(location.toString())) { String childPath = URIUtils.appendPathOrDie(location, status.getName()).toString(); if (status.isDirectory() && !mUfs.deleteDirectory(childPath, DeleteOptions.defaults().setRecursive(true)) || status.isFile() && !mUfs.deleteFile(childPath)) { throw new IOException(String.format("Failed to delete %s", childPath)); } } } else if (!mUfs.mkdirs(location.toString())) { throw new IOException(String.format("Failed to create %s", location)); } // Create a breadcrumb that indicates that the journal folder has been formatted. UnderFileSystemUtils.touch(mUfs, URIUtils.appendPathOrDie(location, ServerConfiguration.get(PropertyKey.MASTER_FORMAT_FILE_PREFIX) + System.currentTimeMillis()) .toString()); }
@Override public TaskResult validate(Map<String, String> optionsMap) { if (!UnderFileSystemUtils.isHdfs(mUfs)) {
UnderFileSystemUtils.touch(ufs, URIUtils.appendPath(mLocation, ServerConfiguration.get(PropertyKey.MASTER_FORMAT_FILE_PREFIX) + System.currentTimeMillis())
@Test public void createOpenEmpty() throws IOException { String testFile = PathUtils.concatPath(mUnderfsAddress, "createOpenEmpty"); createEmptyFile(testFile); byte[] buf = new byte[0]; int bytesRead = mUfs.open(testFile).read(buf); // TODO(adit): Consider making the return value uniform across UFSs if (UnderFileSystemUtils.isHdfs(mUfs)) { assertEquals(-1, bytesRead); } else { assertEquals(0, bytesRead); } }
/** * Constructs a new instance of {@link OSSUnderFileSystem}. * * @param uri the {@link AlluxioURI} for this UFS * @param conf the configuration for this UFS * @param alluxioConf Alluxio configuration * @return the created {@link OSSUnderFileSystem} instance */ public static OSSUnderFileSystem createInstance(AlluxioURI uri, UnderFileSystemConfiguration conf, AlluxioConfiguration alluxioConf) throws Exception { String bucketName = UnderFileSystemUtils.getBucketName(uri); Preconditions.checkArgument(conf.isSet(PropertyKey.OSS_ACCESS_KEY), "Property %s is required to connect to OSS", PropertyKey.OSS_ACCESS_KEY); Preconditions.checkArgument(conf.isSet(PropertyKey.OSS_SECRET_KEY), "Property %s is required to connect to OSS", PropertyKey.OSS_SECRET_KEY); Preconditions.checkArgument(conf.isSet(PropertyKey.OSS_ENDPOINT_KEY), "Property %s is required to connect to OSS", PropertyKey.OSS_ENDPOINT_KEY); String accessId = conf.get(PropertyKey.OSS_ACCESS_KEY); String accessKey = conf.get(PropertyKey.OSS_SECRET_KEY); String endPoint = conf.get(PropertyKey.OSS_ENDPOINT_KEY); ClientConfiguration ossClientConf = initializeOSSClientConfig(alluxioConf); OSSClient ossClient = new OSSClient(endPoint, accessId, accessKey, ossClientConf); return new OSSUnderFileSystem(uri, ossClient, bucketName, conf, alluxioConf); }
@Override public synchronized JournalOutputStream getCheckpointOutputStream(long latestSequenceNumber) throws IOException { if (mCheckpointOutputStream == null) { mCheckpointManager.recover(); LOG.info("Creating tmp checkpoint file: {}", mTempCheckpoint); if (!mUfs.isDirectory(mJournal.getLocation().toString())) { LOG.info("Creating journal folder: {}", mJournal.getLocation()); mUfs.mkdirs(mJournal.getLocation().toString()); } mNextEntrySequenceNumber = latestSequenceNumber + 1; LOG.info("Latest journal sequence number: {} Next journal sequence number: {}", latestSequenceNumber, mNextEntrySequenceNumber); UnderFileSystemUtils.deleteFileIfExists(mUfs, mTempCheckpoint.toString()); mCheckpointOutputStream = new CheckpointOutputStream( new DataOutputStream(mUfs.create(mTempCheckpoint.toString()))); } return mCheckpointOutputStream; }
@Override public UfsFileStatus getFileStatus(String path) throws IOException { String tpath = stripPath(path); File file = new File(tpath); PosixFileAttributes attr = Files.readAttributes(Paths.get(file.getPath()), PosixFileAttributes.class); String contentHash = UnderFileSystemUtils.approximateContentHash(file.length(), file.lastModified()); return new UfsFileStatus(path, contentHash, file.length(), file.lastModified(), attr.owner().getName(), attr.group().getName(), FileUtils.translatePosixPermissionToMode(attr.permissions())); }
UnderFileSystem ufs = UnderFileSystem.Factory.create(job.getTempUfsPath(), ServerConfiguration.global()); UnderFileSystemUtils.touch(ufs, job.getTempUfsPath());
@Override public TaskResult validate(Map<String, String> optionsMap) { if (!UnderFileSystemUtils.isHdfs(mUfs)) {
protected static KodoUnderFileSystem creatInstance(AlluxioURI uri, UnderFileSystemConfiguration conf, AlluxioConfiguration alluxioConf) { String bucketName = UnderFileSystemUtils.getBucketName(uri); Preconditions.checkArgument(conf.isSet(PropertyKey.KODO_ACCESS_KEY), "Property %s is required to connect to Kodo", PropertyKey.KODO_ACCESS_KEY); Preconditions.checkArgument(conf.isSet(PropertyKey.KODO_SECRET_KEY), "Property %s is required to connect to Kodo", PropertyKey.KODO_SECRET_KEY); Preconditions.checkArgument(conf.isSet(PropertyKey.KODO_DOWNLOAD_HOST), "Property %s is required to connect to Kodo", PropertyKey.KODO_DOWNLOAD_HOST); Preconditions.checkArgument(conf.isSet(PropertyKey.KODO_ENDPOINT), "Property %s is required to connect to Kodo", PropertyKey.KODO_ENDPOINT); String accessKey = conf.get(PropertyKey.KODO_ACCESS_KEY); String secretKey = conf.get(PropertyKey.KODO_SECRET_KEY); String endPoint = conf.get(PropertyKey.KODO_ENDPOINT); String souceHost = conf.get(PropertyKey.KODO_DOWNLOAD_HOST); Auth auth = Auth.create(accessKey, secretKey); Configuration configuration = new Configuration(); OkHttpClient.Builder okHttpBuilder = initializeKodoClientConfig(conf); OkHttpClient okHttpClient = okHttpBuilder.build(); KodoClient kodoClient = new KodoClient(auth, bucketName, souceHost, endPoint, configuration, okHttpClient); return new KodoUnderFileSystem(uri, kodoClient, conf, alluxioConf); }
UnderFileSystemUtils.deleteFileIfExists(mUfs, mCheckpoint.toString()); mUfs.renameFile(mTempBackupCheckpoint.toString(), mCheckpoint.toString());
@Override public UfsStatus[] listStatus(String path) throws IOException { path = stripPath(path); File file = new File(path); File[] files = file.listFiles(); if (files != null) { UfsStatus[] rtn = new UfsStatus[files.length]; int i = 0; for (File f : files) { // TODO(adit): do we need extra call for attributes? PosixFileAttributes attr = Files.readAttributes(Paths.get(f.getPath()), PosixFileAttributes.class); short mode = FileUtils.translatePosixPermissionToMode(attr.permissions()); UfsStatus retStatus; if (f.isDirectory()) { retStatus = new UfsDirectoryStatus(f.getName(), attr.owner().getName(), attr.group().getName(), mode, f.lastModified()); } else { String contentHash = UnderFileSystemUtils.approximateContentHash(f.length(), f.lastModified()); retStatus = new UfsFileStatus(f.getName(), contentHash, f.length(), f.lastModified(), attr.owner().getName(), attr.group().getName(), mode); } rtn[i++] = retStatus; } return rtn; } else { return null; } }
UnderFileSystem ufs = UnderFileSystem.Factory.create(job.getTempUfsPath(), ServerConfiguration.global()); UnderFileSystemUtils.touch(ufs, job.getTempUfsPath());
/** * Constructs a new instance of {@link COSUnderFileSystem}. * * @param uri the {@link AlluxioURI} for this UFS * @param conf the configuration for this UFS * @param alluxioConf Alluxio configuration * @return the created {@link COSUnderFileSystem} instance */ public static COSUnderFileSystem createInstance(AlluxioURI uri, UnderFileSystemConfiguration conf, AlluxioConfiguration alluxioConf) throws Exception { String bucketName = UnderFileSystemUtils.getBucketName(uri); Preconditions.checkArgument(conf.isSet(PropertyKey.COS_ACCESS_KEY), "Property %s is required to connect to COS", PropertyKey.COS_ACCESS_KEY); Preconditions.checkArgument(conf.isSet(PropertyKey.COS_SECRET_KEY), "Property %s is required to connect to COS", PropertyKey.COS_SECRET_KEY); Preconditions.checkArgument(conf.isSet(PropertyKey.COS_REGION), "Property %s is required to connect to COS", PropertyKey.COS_REGION); Preconditions.checkArgument(conf.isSet(PropertyKey.COS_APP_ID), "Property %s is required to connect to COS", PropertyKey.COS_APP_ID); String accessKey = conf.get(PropertyKey.COS_ACCESS_KEY); String secretKey = conf.get(PropertyKey.COS_SECRET_KEY); String regionName = conf.get(PropertyKey.COS_REGION); String appId = conf.get(PropertyKey.COS_APP_ID); COSCredentials cred = new BasicCOSCredentials(accessKey, secretKey); ClientConfig clientConfig = createCOSClientConfig(regionName, conf); COSClient client = new COSClient(cred, clientConfig); return new COSUnderFileSystem(uri, client, bucketName, appId, conf, alluxioConf); }
String ufsBlockPath = alluxio.worker.BlockUtils.getUfsBlockPath(ufsClient, blockId); try (CloseableResource<UnderFileSystem> ufsResource = ufsClient.acquireUfsResource()) { alluxio.util.UnderFileSystemUtils.deleteFileIfExists(ufsResource.get(), ufsBlockPath); } catch (Exception e) { LOG.warn("Failed to clean up staging UFS block file {}", ufsBlockPath, e.getMessage());
@Override public UfsStatus getStatus(String path) throws IOException { String tpath = stripPath(path); File file = new File(tpath); PosixFileAttributes attr = Files.readAttributes(Paths.get(file.getPath()), PosixFileAttributes.class); if (file.isFile()) { // Return file status. String contentHash = UnderFileSystemUtils.approximateContentHash(file.length(), file.lastModified()); return new UfsFileStatus(path, contentHash, file.length(), file.lastModified(), attr.owner().getName(), attr.group().getName(), FileUtils.translatePosixPermissionToMode(attr.permissions())); } // Return directory status. return new UfsDirectoryStatus(path, attr.owner().getName(), attr.group().getName(), FileUtils.translatePosixPermissionToMode(attr.permissions()), file.lastModified()); }
UnderFileSystemUtils.touch(ufs, URIUtils.appendPath(mLocation, Configuration.get(PropertyKey.MASTER_FORMAT_FILE_PREFIX) + System.currentTimeMillis()) .toString());