@Override public void check(FileSystem fs) throws Exception { Assert.assertTrue(fs.getStatus(DIR).isPersisted()); Assert.assertTrue(fs.getStatus(INNER_FILE).isPersisted()); } }
@Override public void check(FileSystem fs) throws Exception { assertTrue(fs.getStatus(FILE).isPersisted()); assertTrue(fs.getStatus(NESTED_FILE).isPersisted()); assertTrue(fs.getStatus(NESTED_FILE.getParent()).isPersisted()); } }
@Override public void check(Clients clients) throws Exception { for (AlluxioURI uri : Arrays.asList(PATH, NESTED_PATH, NESTED_PATH.getParent())) { assertTrue(clients.getFileSystemClient().getStatus(uri).isPersisted()); } } }
private void queueNonPersistedRecursive(URIStatus status, Queue<AlluxioURI> toPersist) throws AlluxioException, IOException { AlluxioURI uri = new AlluxioURI(status.getPath()); if (status.isFolder()) { List<URIStatus> statuses = mFileSystem.listStatus(uri); for (URIStatus s : statuses) { queueNonPersistedRecursive(s, toPersist); } } else if (!status.isPersisted()) { toPersist.add(uri); } }
/** * Persists the given file to the under file system. * * @param fs {@link FileSystem} to carry out Alluxio operations * @param fsContext the {@link FileSystemContext} linked to the {@link FileSystem} client * @param uri the uri of the file to persist */ public static void persistFile(final FileSystem fs, final FileSystemContext fsContext, final AlluxioURI uri) throws IOException, TimeoutException, InterruptedException { FileSystemMasterClient client = fsContext.acquireMasterClient(); try { client.scheduleAsyncPersist(uri); } finally { fsContext.releaseMasterClient(client); } CommonUtils.waitFor(String.format("%s to be persisted", uri) , () -> { try { return fs.getStatus(uri).isPersisted(); } catch (Exception e) { Throwables.propagateIfPossible(e); throw new RuntimeException(e); } }, WaitForOptions.defaults().setTimeoutMs(20 * Constants.MINUTE_MS) .setInterval(Constants.SECOND_MS)); }
@Override public void apply(Clients clients) throws Exception { FileSystem fs = clients.getFileSystemClient(); Utils.createFile(fs, PATH); clients.getFileSystemMasterClient().scheduleAsyncPersist(PATH); Utils.createFile(fs, NESTED_PATH); clients.getFileSystemMasterClient().scheduleAsyncPersist(NESTED_PATH); CommonUtils.waitFor("file to be async persisted", () -> { try { return fs.getStatus(PATH).isPersisted() && fs.getStatus(NESTED_PATH).isPersisted(); } catch (Exception e) { throw new RuntimeException(e); } }, WaitForOptions.defaults().setTimeoutMs(10 * Constants.SECOND_MS)); }
if (options.getStatus().isPersisted() || options.getStatus().getPersistenceState().equals("TO_BE_PERSISTED")) { blockWorkerInfo = getEligibleWorkers();
@Override public void apply(FileSystem fs) throws Exception { try (FileOutStream out = fs.createFile(FILE, CreateFilePOptions.newBuilder() .setBlockSizeBytes(Constants.KB).setWriteType(WritePType.ASYNC_THROUGH).build())) { out.write("test".getBytes()); } // Nested file try (FileOutStream out = fs.createFile(NESTED_FILE, CreateFilePOptions.newBuilder().setBlockSizeBytes(Constants.KB) .setWriteType(WritePType.ASYNC_THROUGH).setRecursive(true).build())) { out.write("test".getBytes()); } CommonUtils.waitFor("files to be persisted", () -> { try { return fs.getStatus(FILE).isPersisted() && fs.getStatus(NESTED_FILE).isPersisted(); } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }); }
@Override public void check(FileSystem fs) throws Exception { for (AlluxioURI file : Arrays.asList(PATH, NESTED, THROUGH, TTL)) { assertTrue(fs.exists(file)); } assertEquals(TEST_MODE, new Mode((short) fs.getStatus(MODE).getMode())); assertTrue(fs.getStatus(THROUGH).isPersisted()); assertEquals(TEST_TTL, fs.getStatus(TTL).getTtl()); assertEquals(TtlAction.FREE, fs.getStatus(TTL).getTtlAction()); } }
/** * @param blockId id of the block * @return a {@link Protocol.OpenUfsBlockOptions} based on the block id and options */ public Protocol.OpenUfsBlockOptions getOpenUfsBlockOptions(long blockId) { Preconditions.checkArgument(mStatus.getBlockIds().contains(blockId), "blockId"); boolean readFromUfs = mStatus.isPersisted(); // In case it is possible to fallback to read UFS blocks, also fill in the options. boolean storedAsUfsBlock = mStatus.getPersistenceState().equals("TO_BE_PERSISTED"); readFromUfs = readFromUfs || storedAsUfsBlock; if (!readFromUfs) { return Protocol.OpenUfsBlockOptions.getDefaultInstance(); } long blockStart = BlockId.getSequenceNumber(blockId) * mStatus.getBlockSizeBytes(); BlockInfo info = getBlockInfo(blockId); Protocol.OpenUfsBlockOptions openUfsBlockOptions = Protocol.OpenUfsBlockOptions.newBuilder() .setUfsPath(mStatus.getUfsPath()).setOffsetInFile(blockStart).setBlockSize(info.getLength()) .setMaxUfsReadConcurrency(mProtoOptions.getMaxUfsReadConcurrency()) .setNoCache(!ReadType.fromProto(mProtoOptions.getReadType()).isCache()) .setMountId(mStatus.getMountId()).build(); if (storedAsUfsBlock) { // On client-side, we do not have enough mount information to fill in the UFS file path. // Instead, we unset the ufsPath field and fill in a flag ufsBlock to indicate the UFS file // path can be derived from mount id and the block ID. Also because the entire file is only // one block, we set the offset in file to be zero. openUfsBlockOptions = openUfsBlockOptions.toBuilder().clearUfsPath().setBlockInUfsTier(true) .setOffsetInFile(0).build(); } return openUfsBlockOptions; }
@Override public void check(FileSystem fs) throws Exception { for (AlluxioURI dir : Arrays.asList(DIR, NESTED_DIR, NESTED_NESTED_DIR, RECURSIVE, MODE_DIR, TTL_DIR, COMMON_TTL_DIR, THROUGH_DIR, ALL_OPTS_DIR)) { assertTrue(fs.exists(dir)); } assertEquals(TEST_MODE, new Mode((short) fs.getStatus(MODE_DIR).getMode())); assertEquals((long) TTL, fs.getStatus(TTL_DIR).getTtl()); assertEquals(alluxio.grpc.TtlAction.DELETE, fs.getStatus(TTL_DIR).getTtlAction()); assertEquals((long) TTL, fs.getStatus(COMMON_TTL_DIR).getTtl()); assertEquals(alluxio.grpc.TtlAction.DELETE, fs.getStatus(COMMON_TTL_DIR).getTtlAction()); assertTrue(fs.getStatus(THROUGH_DIR).isPersisted()); assertEquals(TEST_MODE, new Mode((short) fs.getStatus(ALL_OPTS_DIR).getMode())); assertEquals((long) TTL, fs.getStatus(ALL_OPTS_DIR).getTtl()); assertEquals(alluxio.grpc.TtlAction.DELETE, fs.getStatus(ALL_OPTS_DIR).getTtlAction()); assertTrue(fs.getStatus(ALL_OPTS_DIR).isPersisted()); } }
Assert.assertEquals(uriStatus.isCompleted(), fileInfo.isCompleted()); Assert.assertEquals(uriStatus.isFolder(), fileInfo.isFolder()); Assert.assertEquals(uriStatus.isPersisted(), fileInfo.isPersisted()); Assert.assertEquals(uriStatus.isPinned(), fileInfo.isPinned()); Assert.assertEquals(uriStatus.isMountPoint(), fileInfo.isMountPoint());
LOG.debug("Failed to get BlockInStream for block with ID {}, using UFS instead. {}", blockId, e); if (!mStatus.isPersisted()) { LOG.error("Could not obtain data for block with ID {} from Alluxio." + " The block is also not available in the under storage.", blockId);