Refine search
@Override public int read(byte[] buffer, int offset, int length) throws IOException { if (mClosed) { throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage()); } int bytesRead = mInputStream.read(buffer, offset, length); if (bytesRead != -1 && mStatistics != null) { mStatistics.incrementBytesRead(bytesRead); } return bytesRead; }
@Override public FileStatus getFileStatus(Path path) throws IOException { statistics.incrementReadOps(1); System.out.println("STATS: getFileStatus - " + path); checkAccess(); path = path.makeQualified(this); String pathnameAsDir = path.toString() + "/"; MockFile file = findFile(path); if (file != null) return createStatus(file); for (MockFile dir : files) { if (dir.path.toString().startsWith(pathnameAsDir)) { return createDirectory(path); } } for (MockFile dir : globalFiles) { if (dir.path.toString().startsWith(pathnameAsDir)) { return createDirectory(path); } } throw new FileNotFoundException("File " + path + " does not exist"); }
/** * @param file File. * @param append Append flag. * @return Output stream. * @throws IOException If failed. */ private FSDataOutputStream out(File file, boolean append, int bufSize) throws IOException { return new FSDataOutputStream(new BufferedOutputStream(new FileOutputStream(file, append), bufSize < 32 * 1024 ? 32 * 1024 : bufSize), new Statistics(getUri().getScheme())); }
@Override protected Path resolveLink(Path f) throws IOException { statistics.incrementReadOps(1); String target = dfs.getLinkTarget(getPathName(fixRelativePart(f))); if (target == null) { throw new FileNotFoundException("File does not exist: " + f.toString()); } return new Path(target); }
OrcFile.createWriter(new Path(mockPath + "/0_0"), OrcFile.writerOptions(conf).blockPadding(false) .bufferSize(1024).inspector(inspector)); writer = OrcFile.createWriter(new Path(mockPath + "/0_1"), OrcFile.writerOptions(conf).blockPadding(false) .bufferSize(1024).inspector(inspector)); assertEquals(2, splits.length); int readOpsBefore = -1; for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore;
conf.set("fs.defaultFS", "mock:///"); conf.set("fs.mock.impl", MockFileSystem.class.getName()); FileSystem fs = FileSystem.get(conf); MockPath mockPath = new MockPath(fs, "mock:///mocktable7"); conf.set(IOConstants.SCHEMA_EVOLUTION_COLUMNS, MyRow.getColumnNamesProperty()); OrcFile.createWriter(new Path(mockPath + "/0_0"), OrcFile.writerOptions(conf).blockPadding(false) .bufferSize(1024).inspector(inspector)); assertEquals(2, splits.length); int readOpsBefore = -1; for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore;
@Override public FileStatus[] listStatus(Path path) throws IOException { LOG.debug("listStatus({})", path); if (mStatistics != null) { mStatistics.incrementReadOps(1); } AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path)); List<URIStatus> statuses; try { statuses = mFileSystem.listStatus(uri); } catch (FileDoesNotExistException e) { throw new FileNotFoundException(HadoopUtils.getPathWithoutScheme(path)); } catch (AlluxioException e) { throw new IOException(e); } FileStatus[] ret = new FileStatus[statuses.size()]; for (int k = 0; k < statuses.size(); k++) { URIStatus status = statuses.get(k); ret[k] = new FileStatus(status.getLength(), status.isFolder(), getReplica(status), status.getBlockSizeBytes(), status.getLastModificationTimeMs(), status.getCreationTimeMs(), new FsPermission((short) status.getMode()), status.getOwner(), status.getGroup(), new Path(mAlluxioHeader + status.getPath())); } return ret; }
@Override public FSDataOutputStream create(Path path, FsPermission fsPermission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progressable ) throws IOException { statistics.incrementWriteOps(1); checkAccess(); MockFile file = findFile(path); if (file == null) { file = new MockFile(path.toString(), (int) blockSize, new byte[0]); files.add(file); } return new MockOutputStream(file); }
public static Map<String, FileSystem.Statistics> getCombinedFileSystemStatistics() { final List<FileSystem.Statistics> allStats = FileSystem.getAllStatistics(); final Map<String, FileSystem.Statistics> result = new HashMap<>(); for (FileSystem.Statistics statistics : allStats) { final String scheme = statistics.getScheme(); if (result.containsKey(scheme)) { FileSystem.Statistics existing = result.get(scheme); FileSystem.Statistics combined = combineFileSystemStatistics(existing, statistics); result.put(scheme, combined); } else { result.put(scheme, statistics); } } return result; }
@Override protected HdfsDataOutputStream primitiveCreate(Path f, FsPermission absolutePermission, EnumSet<CreateFlag> flag, int bufferSize, short replication, long blockSize, Progressable progress, ChecksumOpt checksumOpt) throws IOException { statistics.incrementWriteOps(1); final DFSOutputStream dfsos = dfs.primitiveCreate( getPathName(fixRelativePart(f)), absolutePermission, flag, true, replication, blockSize, progress, bufferSize, checksumOpt); return dfs.createWrappedOutputStream(dfsos, statistics); }
private DirListingIterator(Path p, PathFilter filter, boolean needLocation) throws IOException { this.p = p; this.src = getPathName(p); this.filter = filter; this.needLocation = needLocation; // fetch the first batch of entries in the directory thisListing = dfs.listPaths(src, HdfsFileStatus.EMPTY_NAME, needLocation); statistics.incrementReadOps(1); if (needLocation) { storageStatistics.incrementOpCounter(OpType.LIST_LOCATED_STATUS); } else { storageStatistics.incrementOpCounter(OpType.LIST_STATUS); } if (thisListing == null) { // the directory does not exist throw new FileNotFoundException("File " + p + " does not exist."); } i = 0; }
@SuppressWarnings("deprecation") @Override protected boolean primitiveMkdir(Path f, FsPermission absolutePermission) throws IOException { statistics.incrementWriteOps(1); return dfs.primitiveMkdir(getPathName(f), absolutePermission); }
/** Get all the existing storage policies */ public BlockStoragePolicy[] getStoragePolicies() throws IOException { statistics.incrementReadOps(1); return dfs.getStoragePolicies(); }
@Override public byte[] getXAttr(Path path, final String name) throws IOException { statistics.incrementReadOps(1); storageStatistics.incrementOpCounter(OpType.GET_XATTR); final Path absF = fixRelativePart(path); return new FileSystemLinkResolver<byte[]>() { @Override public byte[] doCall(final Path p) throws IOException { return dfs.getXAttr(getPathName(p), name); } @Override public byte[] next(final FileSystem fs, final Path p) throws IOException { return fs.getXAttr(p, name); } }.resolve(this, absF); }
/** * Returns number of bytes within blocks with future generation stamp. These * are bytes that will be potentially deleted if we forceExit from safe mode. * * @return number of bytes. */ public long getBytesWithFutureGenerationStamps() throws IOException { statistics.incrementReadOps(1); storageStatistics.incrementOpCounter(OpType.GET_BYTES_WITH_FUTURE_GS); return dfs.getBytesInFutureBlocks(); }
private static FileSystem.Statistics combineFileSystemStatistics(final FileSystem.Statistics s1, final FileSystem.Statistics s2) { FileSystem.Statistics result = new FileSystem.Statistics(s1); result.incrementReadOps(s2.getReadOps()); result.incrementLargeReadOps(s2.getLargeReadOps()); result.incrementWriteOps(s2.getWriteOps()); result.incrementBytesRead(s2.getBytesRead()); result.incrementBytesWritten(s2.getBytesWritten()); return result; }
@Override public int read(long position, byte[] buffer, int offset, int length) throws IOException { if (mClosed) { throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage()); } int bytesRead = mInputStream.positionedRead(position, buffer, offset, length); if (bytesRead != -1 && mStatistics != null) { mStatistics.incrementBytesRead(bytesRead); } return bytesRead; }
@Override public int read() throws IOException { if (mClosed) { throw new IOException(ExceptionMessage.READ_CLOSED_STREAM.getMessage()); } int data = mInputStream.read(); if (data != -1 && mStatistics != null) { mStatistics.incrementBytesRead(1); } return data; }
@Override public FSDataOutputStream append(Path path, int bufferSize, Progressable progressable ) throws IOException { statistics.incrementWriteOps(1); checkAccess(); return create(path, FsPermission.getDefault(), true, bufferSize, (short) 3, 256 * 1024, progressable); }
@Override public FSDataOutputStream append(Path path, int bufferSize, Progressable progressable ) throws IOException { statistics.incrementWriteOps(1); checkAccess(); return create(path, FsPermission.getDefault(), true, bufferSize, (short) 3, 256 * 1024, progressable); }