private boolean readFile(FileSystem fs) throws IOException, AlluxioException { boolean pass = true; for (int i = 0; i < mNumFiles; i++) { AlluxioURI filePath = new AlluxioURI(mFileFolder + "/part-" + i); LOG.debug("Reading data from {}", filePath); FileInStream is = fs.openFile(filePath); URIStatus status = fs.getStatus(filePath); ByteBuffer buf = ByteBuffer.allocate((int) status.getBlockSizeBytes()); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumFiles; k++) { pass = pass && (buf.getInt() == k); } is.close(); } return pass; }
@Override public FileStatus[] listStatus(Path path) throws IOException { LOG.debug("listStatus({})", path); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(uri); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(HadoopUtils.getPathWithoutScheme(path)); } catch (AlluxioException e) { throw new IOException(e); } FileStatus[] ret = new FileStatus[statuses.size()]; for (int k = 0; k < statuses.size(); k++) { URIStatus status = statuses.get(k); ret[k] = new FileStatus(status.getLength(), status.isFolder(), getReplica(status), status.getBlockSizeBytes(), status.getLastModificationTimeMs(), status.getCreationTimeMs(), new FsPermission((short) status.getMode()), status.getOwner(), status.getGroup(), new Path(mAlluxioHeader + status.getPath())); } return ret; }
protected FileInStream(URIStatus status, InStreamOptions options, FileSystemContext context) { AlluxioConfiguration conf = context.getConf(); mPassiveCachingEnabled = conf.getBoolean(PropertyKey.USER_FILE_PASSIVE_CACHE_ENABLED); mBlockWorkerClientReadRetry = conf.getInt(PropertyKey.USER_BLOCK_WORKER_CLIENT_READ_RETRY); mStatus = status; mOptions = options; mBlockStore = AlluxioBlockStore.create(context); mContext = context; mLength = mStatus.getLength(); mBlockSize = mStatus.getBlockSizeBytes(); mPosition = 0; mBlockInStream = null; mCachedPositionedReadStream = null; mLastBlockIdCached = 0; }
/** * {@inheritDoc} * * If the file does not exist in Alluxio, query it from HDFS. */ @Override public FileStatus getFileStatus(Path path) throws IOException { LOG.debug("getFileStatus({})", path); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); URIStatus fileStatus; try { fileStatus = mFileSystem.getStatus(uri); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(e.getMessage()); } catch (AlluxioException e) { throw new IOException(e); } return new FileStatus(fileStatus.getLength(), fileStatus.isFolder(), getReplica(fileStatus), fileStatus.getBlockSizeBytes(), fileStatus.getLastModificationTimeMs(), fileStatus.getCreationTimeMs(), new FsPermission((short) fileStatus.getMode()), fileStatus.getOwner(), fileStatus.getGroup(), new Path(mAlluxioHeader + uri)); }
/** * @param blockId id of the block * @return a {@link Protocol.OpenUfsBlockOptions} based on the block id and options */ public Protocol.OpenUfsBlockOptions getOpenUfsBlockOptions(long blockId) { Preconditions.checkArgument(mStatus.getBlockIds().contains(blockId), "blockId"); boolean readFromUfs = mStatus.isPersisted(); // In case it is possible to fallback to read UFS blocks, also fill in the options. boolean storedAsUfsBlock = mStatus.getPersistenceState().equals("TO_BE_PERSISTED"); readFromUfs = readFromUfs || storedAsUfsBlock; if (!readFromUfs) { return Protocol.OpenUfsBlockOptions.getDefaultInstance(); } long blockStart = BlockId.getSequenceNumber(blockId) * mStatus.getBlockSizeBytes(); BlockInfo info = getBlockInfo(blockId); Protocol.OpenUfsBlockOptions openUfsBlockOptions = Protocol.OpenUfsBlockOptions.newBuilder() .setUfsPath(mStatus.getUfsPath()).setOffsetInFile(blockStart).setBlockSize(info.getLength()) .setMaxUfsReadConcurrency(mProtoOptions.getMaxUfsReadConcurrency()) .setNoCache(!ReadType.fromProto(mProtoOptions.getReadType()).isCache()) .setMountId(mStatus.getMountId()).build(); if (storedAsUfsBlock) { // On client-side, we do not have enough mount information to fill in the UFS file path. // Instead, we unset the ufsPath field and fill in a flag ufsBlock to indicate the UFS file // path can be derived from mount id and the block ID. Also because the entire file is only // one block, we set the offset in file to be zero. openUfsBlockOptions = openUfsBlockOptions.toBuilder().clearUfsPath().setBlockInUfsTier(true) .setOffsetInFile(0).build(); } return openUfsBlockOptions; }
/** * Creates a new instance of {@link alluxio.util.webui.UIFileInfo}. * * @param status underlying {@link URIStatus} */ public UIFileInfo(URIStatus status) { // detect the extended acls boolean hasExtended = status.getAcl().hasExtended() || !status.getDefaultAcl().isEmpty(); mId = status.getFileId(); mName = status.getName(); mAbsolutePath = status.getPath(); mBlockSizeBytes = status.getBlockSizeBytes(); mSize = status.getLength(); mCreationTimeMs = status.getCreationTimeMs(); mLastModificationTimeMs = status.getLastModificationTimeMs(); mInAlluxio = (100 == status.getInAlluxioPercentage()); mInAlluxioPercentage = status.getInAlluxioPercentage(); mIsDirectory = status.isFolder(); mPinned = status.isPinned(); mOwner = status.getOwner(); mGroup = status.getGroup(); mMode = FormatUtils.formatMode((short) status.getMode(), status.isFolder(), hasExtended); mPersistenceState = status.getPersistenceState(); mFileLocations = new ArrayList<>(); }
URIStatus uriStatus = new URIStatus(fileInfo); Assert.assertEquals(uriStatus.getBlockIds(), fileInfo.getBlockIds()); Assert.assertEquals(uriStatus.getBlockSizeBytes(), fileInfo.getBlockSizeBytes()); Assert.assertEquals(uriStatus.getCreationTimeMs(),
private boolean readFile(FileSystem fs) throws IOException, AlluxioException { boolean pass = true; for (int i = 0; i < mNumFiles; i++) { AlluxioURI filePath = new AlluxioURI(mFileFolder + "/part-" + i); LOG.debug("Reading data from {}", filePath); FileInStream is = fs.openFile(filePath); URIStatus status = fs.getStatus(filePath); ByteBuffer buf = ByteBuffer.allocate((int) status.getBlockSizeBytes()); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumFiles; k++) { pass = pass && (buf.getInt() == k); } is.close(); } return pass; }
/** * Creates a new file input stream. * * @param status the file status * @param options the client options */ public FileInStream(URIStatus status, InStreamOptions options) { mStatus = status; mBlockSize = status.getBlockSizeBytes(); mFileLength = status.getLength(); mContext = FileSystemContext.INSTANCE; mAlluxioStorageType = options.getAlluxioStorageType(); mShouldCacheCurrentBlock = mAlluxioStorageType.isStore(); mClosed = false; mLocationPolicy = options.getLocationPolicy(); if (mShouldCacheCurrentBlock) { Preconditions.checkNotNull(options.getLocationPolicy(), PreconditionMessage.FILE_WRITE_LOCATION_POLICY_UNSPECIFIED); } }
@Override public FileStatus[] listStatus(Path path) throws IOException { AlluxioURI tPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); Path hdfsPath = HadoopUtils.getHDFSPath(tPath, mUnderFSAddress); LOG.info("listStatus({}): HDFS Path: {}", path, hdfsPath); if (mStatistics != null) { mStatistics.incrementReadOps(1); } List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(tPath); } catch (AlluxioException e) { throw new IOException(e); } FileStatus[] ret = new FileStatus[statuses.size()]; for (int k = 0; k < statuses.size(); k++) { URIStatus status = statuses.get(k); // TODO(hy): Replicate 3 with the number of disk replications. ret[k] = new FileStatus(status.getLength(), status.isFolder(), 3, status.getBlockSizeBytes(), status.getCreationTimeMs(), status.getCreationTimeMs(), null, null, null, new Path(mAlluxioHeader + status.getPath())); } return ret; }
@Override public FileStatus[] listStatus(Path path) throws IOException { LOG.debug("listStatus({})", path); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(uri); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(HadoopUtils.getPathWithoutScheme(path)); } catch (AlluxioException e) { throw new IOException(e); } FileStatus[] ret = new FileStatus[statuses.size()]; for (int k = 0; k < statuses.size(); k++) { URIStatus status = statuses.get(k); ret[k] = new FileStatus(status.getLength(), status.isFolder(), getReplica(status), status.getBlockSizeBytes(), status.getLastModificationTimeMs(), status.getCreationTimeMs(), new FsPermission((short) status.getMode()), status.getOwner(), status.getGroup(), new Path(mAlluxioHeader + status.getPath())); } return ret; }
/** * {@inheritDoc} * * If the file does not exist in Alluxio, query it from HDFS. */ @Override public FileStatus getFileStatus(Path path) throws IOException { LOG.debug("getFileStatus({})", path); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); URIStatus fileStatus; try { fileStatus = mFileSystem.getStatus(uri); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(e.getMessage()); } catch (AlluxioException e) { throw new IOException(e); } return new FileStatus(fileStatus.getLength(), fileStatus.isFolder(), getReplica(fileStatus), fileStatus.getBlockSizeBytes(), fileStatus.getLastModificationTimeMs(), fileStatus.getCreationTimeMs(), new FsPermission((short) fileStatus.getMode()), fileStatus.getOwner(), fileStatus.getGroup(), new Path(mAlluxioHeader + uri)); }
/** * {@inheritDoc} * * If the file does not exist in Alluxio, query it from HDFS. */ @Override public FileStatus getFileStatus(Path path) throws IOException { AlluxioURI tPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); Path hdfsPath = HadoopUtils.getHDFSPath(tPath, mUnderFSAddress); LOG.info("getFileStatus({}): HDFS Path: {} Alluxio Path: {}{}", path, hdfsPath, mAlluxioHeader, tPath); if (mStatistics != null) { mStatistics.incrementReadOps(1); } URIStatus fileStatus; try { fileStatus = mFileSystem.getStatus(tPath); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(e.getMessage()); } catch (AlluxioException e) { throw new IOException(e); } FileStatus ret = new FileStatus(fileStatus.getLength(), fileStatus.isFolder(), BLOCK_REPLICATION_CONSTANT, fileStatus.getBlockSizeBytes(), fileStatus.getCreationTimeMs(), fileStatus.getCreationTimeMs(), new FsPermission((short) fileStatus.getPermission()), fileStatus.getUserName(), fileStatus.getGroupName(), new Path(mAlluxioHeader + tPath)); return ret; }
/** * Creates a new instance of {@link UIFileInfo}. * * @param status underlying {@link URIStatus} */ public UIFileInfo(URIStatus status) { mId = status.getFileId(); mName = status.getName(); mAbsolutePath = status.getPath(); mBlockSizeBytes = status.getBlockSizeBytes(); mSize = status.getLength(); mCreationTimeMs = status.getCreationTimeMs(); mLastModificationTimeMs = status.getLastModificationTimeMs(); mInAlluxio = (100 == status.getInAlluxioPercentage()); mInAlluxioPercentage = status.getInAlluxioPercentage(); mIsDirectory = status.isFolder(); mPinned = status.isPinned(); mOwner = status.getOwner(); mGroup = status.getGroup(); mMode = FormatUtils.formatMode((short) status.getMode(), status.isFolder()); mPersistenceState = status.getPersistenceState(); mFileLocations = new ArrayList<>(); }