/** * Creates with the default {@link OpenFilePOptions}. * * @param status the file to create the options for * @param alluxioConf Alluxio configuration */ public InStreamOptions(URIStatus status, AlluxioConfiguration alluxioConf) { this(status, OpenFilePOptions.getDefaultInstance(), alluxioConf); }
public alluxio.grpc.OpenFilePOptions getDefaultInstanceForType() { return alluxio.grpc.OpenFilePOptions.getDefaultInstance(); }
@Override public FileInStream openFile(AlluxioURI path) throws FileDoesNotExistException, IOException, AlluxioException { return openFile(path, OpenFilePOptions.getDefaultInstance()); }
/** * <code>optional string fileReadLocationPolicy = 3;</code> */ public Builder clearFileReadLocationPolicy() { bitField0_ = (bitField0_ & ~0x00000004); fileReadLocationPolicy_ = getDefaultInstance().getFileReadLocationPolicy(); onChanged(); return this; } /**
/** * Constructs a new stream for reading a file from HDFS. * * @param fs the file system * @param uri the Alluxio file URI * @param stats filesystem statistics */ public HdfsFileInputStream(FileSystem fs, AlluxioURI uri, Statistics stats) throws IOException { LOG.debug("HdfsFileInputStream({}, {})", uri, stats); mStatistics = stats; try { mInputStream = fs.openFile(uri, OpenFilePOptions.getDefaultInstance()); } catch (FileDoesNotExistException e) { // Transform the Alluxio exception to a Java exception to satisfy the HDFS API contract. throw new FileNotFoundException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(uri)); } catch (AlluxioException e) { throw new IOException(e); } }
/** * Copies a file in the Alluxio filesystem. * * @param srcPath the source {@link AlluxioURI} (has to be a file) * @param dstPath the destination path in the Alluxio filesystem */ private void copyFile(AlluxioURI srcPath, AlluxioURI dstPath) throws AlluxioException, IOException { try (Closer closer = Closer.create()) { OpenFilePOptions openFileOptions = OpenFilePOptions.getDefaultInstance(); FileInStream is = closer.register(mFileSystem.openFile(srcPath, openFileOptions)); FileOutStream os = closer.register(mFileSystem.createFile(dstPath)); try { IOUtils.copy(is, os); } catch (Exception e) { os.cancel(); throw e; } System.out.println(String.format(COPY_SUCCEED_MESSAGE, srcPath, dstPath)); } }
OpenFilePOptions options = OpenFilePOptions.getDefaultInstance(); FileInStream is = closer.register(mFileSystem.openFile(srcPath, options)); FileOutputStream out = closer.register(new FileOutputStream(tmpDst));
@Override protected void runPlainPath(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(path); int numOfBytes = Constants.KB; if (cl.hasOption('c')) { numOfBytes = (int) FormatUtils.parseSpaceSize(cl.getOptionValue('c')); Preconditions.checkArgument(numOfBytes > 0, "specified bytes must be > 0"); } if (status.isFolder()) { throw new IOException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(path)); } OpenFilePOptions options = OpenFilePOptions.getDefaultInstance(); try (FileInStream is = mFileSystem.openFile(path, options)) { byte[] buf = new byte[numOfBytes]; long bytesToRead; if (status.getLength() > numOfBytes) { bytesToRead = numOfBytes; } else { bytesToRead = status.getLength(); } is.skip(status.getLength() - bytesToRead); int read = is.read(buf); if (read != -1) { System.out.write(buf, 0, read); } } }
@Override protected void runPlainPath(AlluxioURI plainPath, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(plainPath); if (status.isFolder()) { throw new IOException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(plainPath)); } OpenFilePOptions options = OpenFilePOptions.getDefaultInstance(); try (FileInStream is = mFileSystem.openFile(plainPath, options)) { long bytesToRead; if (status.getLength() > mNumOfBytes) { bytesToRead = mNumOfBytes; } else { bytesToRead = status.getLength(); } byte[] buf = new byte[(int) bytesToRead]; int read = is.read(buf); if (read != -1) { System.out.write(buf, 0, read); } } }
@Override protected void runPlainPath(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(path); if (status.isFolder()) { throw new FileDoesNotExistException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(path)); } OpenFilePOptions options = OpenFilePOptions.getDefaultInstance(); byte[] buf = new byte[Constants.MB]; try (FileInStream is = mFileSystem.openFile(path, options)) { int read = is.read(buf); while (read != -1) { System.out.write(buf, 0, read); read = is.read(buf); } } }
@Test public void getInStreamMissingBlock() throws Exception { URIStatus dummyStatus = new URIStatus( new FileInfo().setPersisted(false).setBlockIds(Collections.singletonList(0L))); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList()); mException.expect(NotFoundException.class); mException.expectMessage("unavailable in both Alluxio and UFS"); mBlockStore.getInStream(BLOCK_ID, options).getAddress(); }
private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, Map<Integer, Long> failedWorkers, int expectedWorker) throws Exception { WorkerNetAddress[] workers = new WorkerNetAddress[workerCount]; Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i))); BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID) .setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList())); URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(isPersisted) .setBlockIds(Collections.singletonList(BLOCK_ID)) .setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info)))); BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class); when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg .getArgumentAt(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next() .getNetAddress()); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); options.setUfsReadLocationPolicy(mockPolicy); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info); when(mMasterClient.getWorkerInfoList()).thenReturn(Arrays.stream(workers) .map(x -> new WorkerInfo().setAddress(x)).collect((Collectors.toList()))); Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream() .map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses); assertEquals(workers[expectedWorker], inStream.getAddress()); } }
@Test public void getInStreamNoWorkers() throws Exception { URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L))); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo()); when(mMasterClient.getWorkerInfoList()).thenReturn(Collections.emptyList()); mException.expect(UnavailableException.class); mException.expectMessage("No Alluxio worker available"); mBlockStore.getInStream(BLOCK_ID, options).getAddress(); }
/** * Tests for the {@link BaseFileSystem#openFile(AlluxioURI, OpenFilePOptions)} method to * complete successfully. */ @Test public void openFile() throws Exception { AlluxioURI file = new AlluxioURI("/file"); URIStatus status = new URIStatus(new FileInfo()); GetStatusPOptions getStatusOptions = GetStatusPOptions.getDefaultInstance(); when(mFileSystemMasterClient.getStatus(file, getStatusOptions)).thenReturn(status); mFileSystem.openFile(file, OpenFilePOptions.getDefaultInstance()); verify(mFileSystemMasterClient).getStatus(file, getStatusOptions); verifyFilesystemContextAcquiredAndReleased(); }
/** * Ensures that an exception is propagated successfully when opening a file. */ @Test public void openException() throws Exception { AlluxioURI file = new AlluxioURI("/file"); GetStatusPOptions getStatusOptions = GetStatusPOptions.getDefaultInstance(); when(mFileSystemMasterClient.getStatus(file, getStatusOptions)).thenThrow(EXCEPTION); try { mFileSystem.openFile(file, OpenFilePOptions.getDefaultInstance()); fail(SHOULD_HAVE_PROPAGATED_MESSAGE); } catch (Exception e) { assertSame(EXCEPTION, e); } verifyFilesystemContextAcquiredAndReleased(); }
.getNetAddress()); InStreamOptions options = new InStreamOptions(dummyStatus, OpenFilePOptions.getDefaultInstance(), sConf); options.setUfsReadLocationPolicy(mockPolicy); when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
public Builder mergeFrom(alluxio.grpc.OpenFilePOptions other) { if (other == alluxio.grpc.OpenFilePOptions.getDefaultInstance()) return this; if (other.hasReadType()) { setReadType(other.getReadType()); } if (other.hasMaxUfsReadConcurrency()) { setMaxUfsReadConcurrency(other.getMaxUfsReadConcurrency()); } if (other.hasFileReadLocationPolicy()) { bitField0_ |= 0x00000004; fileReadLocationPolicy_ = other.fileReadLocationPolicy_; onChanged(); } if (other.hasHashingNumberOfShards()) { setHashingNumberOfShards(other.getHashingNumberOfShards()); } if (other.hasCommonOptions()) { mergeCommonOptions(other.getCommonOptions()); } this.mergeUnknownFields(other.unknownFields); onChanged(); return this; }