@Override protected InputStream open(long offset) throws IOException { TachyonFS tfs = (TachyonFS) (Persist.I[Value.TACHYON].createClient(clientURI)); InputStream is = tfs.getFile(fpath).getInStream(ReadType.NO_CACHE); is.skip(offset); return is; } }
/** * {@inheritDoc} * * If the file does not exist in Tachyon, query it from HDFS. */ @Override public FileStatus getFileStatus(Path path) throws IOException { TachyonURI tPath = new TachyonURI(Utils.getPathWithoutScheme(path)); Path hdfsPath = Utils.getHDFSPath(tPath, mUnderFSAddress); LOG.info("getFileStatus(" + path + "): HDFS Path: " + hdfsPath + " TPath: " + mTachyonHeader + tPath); if (mStatistics != null) { mStatistics.incrementReadOps(1); } TachyonFile file; try { file = mTFS.getFile(tPath); } catch (IOException ioe) { LOG.info("File does not exist: " + path); throw new FileNotFoundException("File does not exist: " + path); } FileStatus ret = new FileStatus(file.length(), file.isDirectory(), file.getDiskReplication(), file.getBlockSizeByte(), file.getCreationTimeMs(), file.getCreationTimeMs(), null, null, null, new Path(mTachyonHeader + tPath)); return ret; }
+ " is not found."); mTachyonFile.setUFSConf(mHadoopConf); mTachyonFileInputStream = mTachyonFile.getInStream(ReadType.CACHE);
@Override public FSDataOutputStream append(Path cPath, int bufferSize, Progressable progress) throws IOException { LOG.info("append(" + cPath + ", " + bufferSize + ", " + progress + ")"); if (mStatistics != null) { mStatistics.incrementWriteOps(1); } TachyonURI path = new TachyonURI(Utils.getPathWithoutScheme(cPath)); long fileId = mTFS.getFileId(path); TachyonFile file = mTFS.getFile(fileId); if (file.length() > 0) { LOG.warn("This maybe an error."); } return new FSDataOutputStream(file.getOutStream(), mStatistics); }
file.setUFSConf(getConf()); return new FSDataOutputStream(file.getOutStream(), mStatistics);
/** * Returns the {@code OutStream} of this file, use the specified write type. Always return a * {@code FileOutStream}. * * @param writeType the OutStream's write type which is unused * * @return the OutStream * @throws IOException when an event that prevents the operation from completing is encountered */ public FileOutStream getOutStream(WriteType writeType) throws IOException { return getOutStream(); }
/** * Returns whether this file is complete or not * * @return true if this file is complete, false otherwise * @throws IOException if the underlying file does not exist or its metadata is corrupted */ public boolean isCompleted() throws IOException { return getCachedFileStatus().isCompleted || getUnCachedFileStatus().isCompleted; }
/** * Returns the net address of all the location hosts * * @return the list of those net address, in String * @throws IOException if the underlying file does not exist or its metadata is corrupted */ public List<String> getLocationHosts() throws IOException { List<String> ret = Lists.newArrayList(); if (getNumberOfBlocks() > 0) { // add tachyon locations first List<BlockLocation> blockLocations = getClientBlockInfo(0).getBlockInfo().getLocations(); if (blockLocations != null) { for (BlockLocation location : blockLocations) { ret.add(location.workerAddress.host); } } // under FS locations List<NetAddress> underFsLocations = getClientBlockInfo(0).getUfsLocations(); if (underFsLocations != null) { for (NetAddress location : underFsLocations) { ret.add(location.host); } } } return ret; }
/** * @return true if this is a directory, false otherwise * @throws IOException if the underlying file does not exist or its metadata is corrupted */ public boolean isDirectory() throws IOException { return getCachedFileStatus().isFolder; }
/** * Promotes block back to top layer after access. * * @param blockIndex the index of the block * @return true if success, false otherwise * @throws IOException if the underlying file does not exist or its metadata is corrupted */ public boolean promoteBlock(int blockIndex) throws IOException { FileBlockInfo fileBlockInfo = getClientBlockInfo(blockIndex); return mTachyonFS.promoteBlock(fileBlockInfo.blockInfo.getBlockId()); }
/** * Gets <code>TachyonFile</code> based on the file id. If useCachedMetadata, this will not see * changes to the file's pin setting, or other dynamic properties. * * @param fid the file id * @param useCachedMetadata whether to use cached metadata * @return TachyonFile of the file id, or null if the file does not exist * @throws IOException if the underlying master RPC fails */ public synchronized TachyonFile getFile(long fid, boolean useCachedMetadata) throws IOException { FileInfo fileInfo = getFileStatus(fid, TachyonURI.EMPTY_URI, useCachedMetadata); if (fileInfo == null) { return null; } return new TachyonFile(this, fid, mTachyonConf); }
@Override public boolean equals(Object obj) { if ((obj != null) && (obj instanceof TachyonFile)) { return compareTo((TachyonFile) obj) == 0; } return false; }
private void write(TachyonFS tachyonClient) throws IOException { RawTable rawTable = tachyonClient.getRawTable(mTablePath); LOG.debug("Writing data..."); for (int column = 0; column < COLS; column ++) { RawColumn rawColumn = rawTable.getRawColumn(column); if (!rawColumn.createPartition(0)) { throw new IOException("Failed to create partition in table " + mTablePath + " under column " + column); } ByteBuffer buf = ByteBuffer.allocate(mDataLength * 4); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mDataLength; k ++) { buf.putInt(k); } buf.flip(); TachyonFile tFile = rawColumn.getPartition(0); FileOutStream os = tFile.getOutStream(mWriteType); os.write(buf.array()); os.close(); } }
/** * Returns the block size of this file. * * @return the block size in bytes * @throws IOException if the underlying file does not exist or its metadata is corrupted */ public long getBlockSizeByte() throws IOException { return getCachedFileStatus().getBlockSizeBytes(); }
/** * Returns the local filename for the block if that file exists on the local file system. This is * an alpha power-api feature for applications that want short-circuit-read files directly. There * is no guarantee that the file still exists after this call returns, as Tachyon may evict blocks * from memory at any time. * * @param blockIndex The index of the block in the file * @return filename on local file system or null if file not present on local file system * @throws IOException if the underlying file does not exist or its metadata is corrupted */ public String getLocalFilename(int blockIndex) throws IOException { FileBlockInfo fileBlockInfo = getClientBlockInfo(blockIndex); long blockId = fileBlockInfo.blockInfo.getBlockId(); int blockLockId = mTachyonFS.getBlockLockId(); String filename = mTachyonFS.lockBlock(blockId, blockLockId); if (filename != null) { mTachyonFS.unlockBlock(blockId, blockLockId); } return filename; }
/** * Gets <code>TachyonFile</code> based on the path. If useCachedMetadata is true, this will not * see changes to the file's pin setting, or other dynamic properties. * * @param path file path. * @param useCachedMetadata whether to use the file metadata cache * @return TachyonFile of the path, or null if the file does not exist * @throws IOException if the underlying master RPC fails */ public synchronized TachyonFile getFile(TachyonURI path, boolean useCachedMetadata) throws IOException { validateUri(path); FileInfo fileInfo = getFileStatus(IdUtils.INVALID_FILE_ID, path, useCachedMetadata); if (fileInfo == null) { return null; } return new TachyonFile(this, fileInfo.getFileId(), mTachyonConf); }
@Override public byte[] load(Value v) { Key k = v._key; // key for value if (k._kb[0] != Key.DVEC) throw H2O.unimpl(); // Load only from values stored in vector long skip = FileVec.chunkOffset(k); // Compute skip for this value long start_io_ms = System.currentTimeMillis(); final byte[] b = MemoryManager.malloc1(v._max); String[] keyComp = decodeKey(k); String clientUri = keyComp[0]; String fpath = keyComp[1]; TachyonFS tfs = null; InputStream is = null; try { tfs = (TachyonFS) (Persist.I[Value.TACHYON].createClient(clientUri)); long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats is = tfs.getFile(fpath).getInStream(ReadType.NO_CACHE); ByteStreams.skipFully(is, skip); ByteStreams.readFully(is, b); TimeLine.record_IOclose(start_ns, start_io_ms, 1/* read */, v._max, Value.TACHYON); return b; } catch (IOException e) { throw new RuntimeException(Log.err("File load failed: ", e)); } finally { if (is!=null) Utils.close(is); } }
/** * Returns the creation time of this file * * @return the creation time, in milliseconds * @throws IOException if the underlying file does not exist or its metadata is corrupted */ public long getCreationTimeMs() throws IOException { return getCachedFileStatus().getCreationTimeMs(); }
private boolean read(TachyonFS tachyonClient) throws IOException { boolean pass = true; LOG.debug("Reading data..."); RawTable rawTable = tachyonClient.getRawTable(mId); ByteBuffer metadata = rawTable.getMetadata(); LOG.debug("Metadata: "); metadata.order(ByteOrder.nativeOrder()); for (int k = -mMetadataLength; k < 0; k ++) { pass = pass && (metadata.getInt() == k); } for (int column = 0; column < COLS; column ++) { RawColumn rawColumn = rawTable.getRawColumn(column); TachyonFile tFile = rawColumn.getPartition(0); FileInStream is = tFile.getInStream(mReadType); ByteBuffer buf = ByteBuffer.allocate(mDataLength * 4); is.read(buf.array()); buf.order(ByteOrder.nativeOrder()); for (int k = 0; k < mDataLength; k ++) { pass = pass && (buf.getInt() == k); } is.close(); } return pass; }