/** * Constructs a new stream for reading a file from HDFS. * * @param fs the file system * @param uri the Alluxio file URI * @param stats filesystem statistics */ public HdfsFileInputStream(FileSystem fs, AlluxioURI uri, Statistics stats) throws IOException { LOG.debug("HdfsFileInputStream({}, {})", uri, stats); mStatistics = stats; try { mInputStream = fs.openFile(uri, OpenFilePOptions.getDefaultInstance()); } catch (FileDoesNotExistException e) { // Transform the Alluxio exception to a Java exception to satisfy the HDFS API contract. throw new FileNotFoundException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(uri)); } catch (AlluxioException e) { throw new IOException(e); } }
/** * Reads a file. * * @param count the count to determine the filename * @throws Exception if it fails to read */ private static void readFile(CyclicBarrier barrier, AtomicLong runTime, int count, AlluxioConfiguration alluxioConf) throws Exception { FileSystem fileSystem = FileSystem.Factory.get(alluxioConf); byte[] buffer = new byte[(int) Math.min(sFileSize, 4 * Constants.MB)]; barrier.await(); long startTime = System.nanoTime(); try (FileInStream inStream = fileSystem.openFile(filename(count))) { while (inStream.read(buffer) != -1) { } } runTime.addAndGet(System.nanoTime() - startTime); }
/** * Copies a file in the Alluxio filesystem. * * @param srcPath the source {@link AlluxioURI} (has to be a file) * @param dstPath the destination path in the Alluxio filesystem */ private void copyFile(AlluxioURI srcPath, AlluxioURI dstPath) throws AlluxioException, IOException { try (Closer closer = Closer.create()) { OpenFilePOptions openFileOptions = OpenFilePOptions.getDefaultInstance(); FileInStream is = closer.register(mFileSystem.openFile(srcPath, openFileOptions)); FileOutStream os = closer.register(mFileSystem.createFile(dstPath)); try { IOUtils.copy(is, os); } catch (Exception e) { os.cancel(); throw e; } System.out.println(String.format(COPY_SUCCEED_MESSAGE, srcPath, dstPath)); } }
private boolean readFile(FileSystem fileSystem) throws IOException, AlluxioException { boolean pass = true; LOG.debug("Reading data..."); final long startTimeMs = CommonUtils.getCurrentMs(); FileInStream is = fileSystem.openFile(mFilePath, mReadOptions); ByteBuffer buf = ByteBuffer.allocate((int) is.remaining()); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < NUMBERS; k++) { pass = pass && (buf.getInt() == k); } is.close(); LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "readFile file " + mFilePath)); return pass; } }
/** * Calculates the md5 checksum for a file. * * @param filePath The {@link AlluxioURI} path of the file calculate MD5 checksum on * @return A string representation of the file's MD5 checksum */ private String calculateChecksum(AlluxioURI filePath) throws AlluxioException, IOException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build(); try (FileInStream fis = mFileSystem.openFile(filePath, options)) { return DigestUtils.md5Hex(fis); } }
private boolean readFile(FileSystem fs) throws IOException, AlluxioException { boolean pass = true; for (int i = 0; i < mNumFiles; i++) { AlluxioURI filePath = new AlluxioURI(mFileFolder + "/part-" + i); LOG.debug("Reading data from {}", filePath); FileInStream is = fs.openFile(filePath); URIStatus status = fs.getStatus(filePath); ByteBuffer buf = ByteBuffer.allocate((int) status.getBlockSizeBytes()); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mNumFiles; k++) { pass = pass && (buf.getInt() == k); } is.close(); } return pass; }
mFileSystem.openFile(new AlluxioURI(sFileName + (pId + sBaseFileNumber))); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); final long startTimeMs = System.currentTimeMillis(); InputStream is = mFileSystem.openFile(new AlluxioURI(sFileName + (pId + sBaseFileNumber))); long len = sBlocksPerFile * sBlockSizeBytes; final long startTimeMs = System.currentTimeMillis(); InputStream is = mFileSystem.openFile(new AlluxioURI(sFileName + (pId + sBaseFileNumber))); for (int i = 0; i < sBlocksPerFile; i++) { is.read(mBuf.array());
private static void basicAlluxioTests() throws Exception { LOG.info("Running tests"); FileSystem fs = FileSystem.Factory.get(sConf); int listSize = fs.listStatus(new AlluxioURI("/")).size(); if (listSize != 1) { throw new RuntimeException("Expected 1 path to exist at the root, but found " + listSize); } FileOutStream outStream = fs.createFile(new AlluxioURI("/test")); outStream.write("abc".getBytes()); outStream.close(); FileInStream inStream = fs.openFile(new AlluxioURI("/test")); String result = IOUtils.toString(inStream); if (!result.equals("abc")) { throw new RuntimeException("Expected abc but got " + result); } LOG.info("Tests passed"); }
FileInStream is = closer.register(mFileSystem.openFile(srcPath, options)); FileOutputStream out = closer.register(new FileOutputStream(tmpDst)); byte[] buf = new byte[64 * Constants.MB];
private boolean read(FileSystem alluxioClient) throws IOException, AlluxioException { OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(mReadType.toProto()).build(); boolean pass = true; long startTimeMs = CommonUtils.getCurrentMs(); try (DataInputStream input = new DataInputStream(alluxioClient.openFile(mFilePath, options))) { int length = input.readInt(); for (int i = 0; i < length; i++) { if (input.readInt() != i) { pass = false; break; } } } LOG.info(FormatUtils.formatTimeTakenMs(startTimeMs, "readFile file " + mFilePath)); return pass; } }
@Override protected void runPlainPath(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(path); int numOfBytes = Constants.KB; if (cl.hasOption('c')) { numOfBytes = (int) FormatUtils.parseSpaceSize(cl.getOptionValue('c')); Preconditions.checkArgument(numOfBytes > 0, "specified bytes must be > 0"); } if (status.isFolder()) { throw new IOException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(path)); } OpenFilePOptions options = OpenFilePOptions.getDefaultInstance(); try (FileInStream is = mFileSystem.openFile(path, options)) { byte[] buf = new byte[numOfBytes]; long bytesToRead; if (status.getLength() > numOfBytes) { bytesToRead = numOfBytes; } else { bytesToRead = status.getLength(); } is.skip(status.getLength() - bytesToRead); int read = is.read(buf); if (read != -1) { System.out.write(buf, 0, read); } } }
@Override protected void runPlainPath(AlluxioURI plainPath, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(plainPath); if (status.isFolder()) { throw new IOException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(plainPath)); } OpenFilePOptions options = OpenFilePOptions.getDefaultInstance(); try (FileInStream is = mFileSystem.openFile(plainPath, options)) { long bytesToRead; if (status.getLength() > mNumOfBytes) { bytesToRead = mNumOfBytes; } else { bytesToRead = status.getLength(); } byte[] buf = new byte[(int) bytesToRead]; int read = is.read(buf); if (read != -1) { System.out.write(buf, 0, read); } } }
@Override protected void runPlainPath(AlluxioURI path, CommandLine cl) throws AlluxioException, IOException { URIStatus status = mFileSystem.getStatus(path); if (status.isFolder()) { throw new FileDoesNotExistException(ExceptionMessage.PATH_MUST_BE_FILE.getMessage(path)); } OpenFilePOptions options = OpenFilePOptions.getDefaultInstance(); byte[] buf = new byte[Constants.MB]; try (FileInStream is = mFileSystem.openFile(path, options)) { int read = is.read(buf); while (read != -1) { System.out.write(buf, 0, read); read = is.read(buf); } } }
@Before public void before() throws Exception { AlluxioConfiguration conf = ConfigurationTestUtils.defaults(); mMockFileSystem = Mockito.mock(FileSystem.class); mMockFileSystemContext = PowerMockito.mock(FileSystemContext.class); when(mMockFileSystemContext.getClientContext()) .thenReturn(ClientContext.create(conf)); when(mMockFileSystemContext.getConf()) .thenReturn(conf); mMockInStream = new MockFileInStream(mMockFileSystemContext, TEST_SOURCE_CONTENTS, conf); when(mMockFileSystem.openFile(new AlluxioURI(TEST_SOURCE))).thenReturn(mMockInStream); mMockOutStream = new MockFileOutStream(mMockFileSystemContext); when(mMockFileSystem.createFile(eq(new AlluxioURI(TEST_DESTINATION)), any(CreateFilePOptions.class))).thenReturn(mMockOutStream); mMockUfsManager = Mockito.mock(UfsManager.class); }
FileInStream is = mFileSystem.openFile(uri); synchronized (mOpenFiles) { mOpenFiles.add(new OpenFileEntry(mNextOpenFileId, path, is, null));
FileInStream in = closer.register(mFileSystem.openFile(filePath, options)); byte[] buf = new byte[8 * Constants.MB]; while (in.read(buf) != -1) {
/** * Loads a file into Alluxio by reading it. * * @param fs a {@link FileSystem} * @param fileName the name of the file to load */ public static void loadFile(FileSystem fs, String fileName) { try (FileInStream is = fs.openFile(new AlluxioURI(fileName), OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE).build())) { IOUtils.copy(is, ByteStreams.nullOutputStream()); } catch (IOException | AlluxioException e) { throw new RuntimeException(e); } }
/** * @param command the move command to execute * @param writeType the write type to use for the moved file * @param fileSystem the Alluxio file system */ private static void move(MoveCommand command, WritePType writeType, FileSystem fileSystem) throws Exception { String source = command.getSource(); String destination = command.getDestination(); LOG.debug("Moving {} to {}", source, destination); CreateFilePOptions createOptions = CreateFilePOptions.newBuilder().setWriteType(writeType).build(); try (FileOutStream out = fileSystem.createFile(new AlluxioURI(destination), createOptions)) { try (FileInStream in = fileSystem.openFile(new AlluxioURI(source))) { IOUtils.copy(in, out); } catch (Throwable t) { try { out.cancel(); } catch (Throwable t2) { t.addSuppressed(t2); } throw t; } } fileSystem.delete(new AlluxioURI(source)); }
/** * Ensures that an exception is propagated successfully when opening a file. */ @Test public void openException() throws Exception { AlluxioURI file = new AlluxioURI("/file"); GetStatusPOptions getStatusOptions = GetStatusPOptions.getDefaultInstance(); when(mFileSystemMasterClient.getStatus(file, getStatusOptions)).thenThrow(EXCEPTION); try { mFileSystem.openFile(file, OpenFilePOptions.getDefaultInstance()); fail(SHOULD_HAVE_PROPAGATED_MESSAGE); } catch (Exception e) { assertSame(EXCEPTION, e); } verifyFilesystemContextAcquiredAndReleased(); }
/** * Tests for the {@link BaseFileSystem#openFile(AlluxioURI, OpenFilePOptions)} method to * complete successfully. */ @Test public void openFile() throws Exception { AlluxioURI file = new AlluxioURI("/file"); URIStatus status = new URIStatus(new FileInfo()); GetStatusPOptions getStatusOptions = GetStatusPOptions.getDefaultInstance(); when(mFileSystemMasterClient.getStatus(file, getStatusOptions)).thenReturn(status); mFileSystem.openFile(file, OpenFilePOptions.getDefaultInstance()); verify(mFileSystemMasterClient).getStatus(file, getStatusOptions); verifyFilesystemContextAcquiredAndReleased(); }