AlluxioURI outputPath = new AlluxioURI("/mnt/hdfs/hello.txt"); AlluxioURI hdfsPath = new AlluxioURI(args[0]); FileSystem fileSystem = FileSystem.Factory.get(alluxioConf);
/** * Writes a file. * * @param count the count to determine the filename * @throws Exception if it fails to write */ private static void writeFile(CyclicBarrier barrier, AtomicLong runtime, int count) throws Exception { FileSystem fileSystem = FileSystem.Factory.get(); byte[] buffer = new byte[(int) Math.min(sFileSize, 4 * Constants.MB)]; Arrays.fill(buffer, (byte) 'a'); AlluxioURI path = filename(count); if (fileSystem.exists(path)) { fileSystem.delete(path); } barrier.await(); long startTime = System.nanoTime(); long bytesWritten = 0; try (FileOutStream outStream = fileSystem.createFile(path)) { while (bytesWritten < sFileSize) { outStream.write(buffer, 0, (int) Math.min(buffer.length, sFileSize - bytesWritten)); bytesWritten += buffer.length; } } runtime.addAndGet(System.nanoTime() - startTime); }
/** * Constructs a new stream for reading a file from HDFS. * * @param context the file system context * @param uri the Alluxio file URI * @param stats filesystem statistics */ public HdfsFileInputStream(FileSystemContext context, AlluxioURI uri, Statistics stats) throws IOException { LOG.debug("HdfsFileInputStream({}, {})", uri, stats); mStatistics = stats; FileSystem fs = FileSystem.Factory.get(context); try { mInputStream = fs.openFile(uri, OpenFileOptions.defaults()); } catch (FileDoesNotExistException e) { // Transform the Alluxio exception to a Java exception to satisfy the HDFS API contract. throw new FileNotFoundException(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(uri)); } catch (AlluxioException e) { throw new IOException(e); } }
/** * Reads a file. * * @param count the count to determine the filename * @throws Exception if it fails to read */ private static void readFile(CyclicBarrier barrier, AtomicLong runTime, int count) throws Exception { FileSystem fileSystem = FileSystem.Factory.get(); byte[] buffer = new byte[(int) Math.min(sFileSize, 4 * Constants.MB)]; barrier.await(); long startTime = System.nanoTime(); try (FileInStream inStream = fileSystem.openFile(filename(count))) { while (inStream.read(buffer) != -1) { } } runTime.addAndGet(System.nanoTime() - startTime); }
private static void basicAlluxioTests() throws Exception { LOG.info("Running tests"); FileSystem fs = FileSystem.Factory.get(); int listSize = fs.listStatus(new AlluxioURI("/")).size(); if (listSize != 1) { throw new RuntimeException("Expected 1 path to exist at the root, but found " + listSize); } FileOutStream outStream = fs.createFile(new AlluxioURI("/test")); outStream.write("abc".getBytes()); outStream.close(); FileInStream inStream = fs.openFile(new AlluxioURI("/test")); String result = IOUtils.toString(inStream); if (!result.equals("abc")) { throw new RuntimeException("Expected abc but got " + result); } LOG.info("Tests passed"); }
/** * Sets the file system and context. Contexts with the same subject are shared among file systems * to reduce resource usage such as the metrics heartbeat. */ private void updateFileSystemAndContext() { Subject subject = getHadoopSubject(); if (subject != null) { LOG.debug("Using Hadoop subject: {}", subject); mContext = FileSystemContext.get(subject); mFileSystem = FileSystem.Factory.get(mContext); } else { LOG.debug("No Hadoop subject. Using FileSystem Context without subject."); mContext = FileSystemContext.get(); mFileSystem = FileSystem.Factory.get(mContext); } }
/** * @param id the id of the worker * @param left the id of the worker on the left * @param right the id of the worker on the right * @param buf the buffer to read */ public AlluxioReadWorker(int id, int left, int right, ByteBuffer buf) throws IOException { super(id, left, right, buf); mFileSystem = FileSystem.Factory.get(); }
@Override public Boolean call() throws Exception { FileSystem alluxioClient = FileSystem.Factory.get(); write(alluxioClient); return read(alluxioClient); }
@Override public Boolean call() throws Exception { FileSystem fs = FileSystem.Factory.get(); writeFile(fs); return readFile(fs); }
/** * Constructs a new {@link RunOperation} object. */ public RunOperation() { mFileSystem = FileSystem.Factory.get(); }
@Override public void init() throws ServletException { super.init(); getServletContext().setAttribute(ALLUXIO_PROXY_SERVLET_RESOURCE_KEY, proxyProcess); getServletContext() .setAttribute(FILE_SYSTEM_SERVLET_RESOURCE_KEY, FileSystem.Factory.get()); getServletContext().setAttribute(STREAM_CACHE_SERVLET_RESOURCE_KEY, new StreamCache(Configuration.getMs(PropertyKey.PROXY_STREAM_CACHE_TIMEOUT_MS))); } };
/** * Returns a {@link FileSystem} client using a specific context. This client does not need to be * closed directly, but can be closed by calling {@link #close()} on this object. * * @param context the FileSystemContext to use * @return a {@link FileSystem} client */ public FileSystem getClient(FileSystemContext context) throws IOException { final FileSystem fs = FileSystem.Factory.get(context); mClients.add(fs); return fs; }
AlluxioUtils() { try { loadConfig(); } catch (BootstrapException e) { e.printStackTrace(); } this.fileSystem = FileSystem.Factory.get(); }
/** * @return a client for interacting with the cluster */ public synchronized FileSystem getFileSystemClient() { Preconditions.checkState(mState == State.STARTED, "must be in the started state to get an fs client, but state was %s", mState); MasterInquireClient inquireClient = getMasterInquireClient(); return Factory.get(mCloser.register(FileSystemContext.create(null, inquireClient))); }
@Override public Boolean call() throws Exception { FileSystem fs = FileSystem.Factory.get(); writeFile(fs); return readFile(fs); }
/** * @param id the id of the worker * @param left the id of the worker on the left * @param right the id of the worker on the right * @param buf the buffer to write */ public AlluxioWriterWorker(int id, int left, int right, ByteBuffer buf) throws IOException { super(id, left, right, buf); mFileSystem = FileSystem.Factory.get(); }
/** * Returns a {@link FileSystem} client. This client does not need to be * closed directly, but can be closed by calling {@link #close()} on this object. * * @return a {@link FileSystem} client */ public FileSystem getClient() throws IOException { final FileSystem fs = FileSystem.Factory.get(); mClients.add(fs); return fs; }
sResultPrefix = "AlluxioFilesWriteTest " + sResultPrefix; LOG.info(sResultPrefix); AlluxioTest(true /* write */, FileSystem.Factory.get(fsContext)); } else if (testCase == 2 || testCase == 9) { sResultPrefix = "AlluxioFilesReadTest " + sResultPrefix; LOG.info(sResultPrefix); sAlluxioStreamingRead = (9 == testCase); AlluxioTest(false /* read */, FileSystem.Factory.get(fsContext)); } else if (testCase == 3) { sResultPrefix = "RamFile Write " + sResultPrefix;
sFileSystem = FileSystem.Factory.get(sConf); try { sFileSystem.delete(new AlluxioURI(sTestDir));
mFileSystem = FileSystem.Factory.get(mFsContext);