FsVolumeImpl build() throws IOException { if (sd.getStorageLocation().getStorageType() == StorageType.PROVIDED) { return new ProvidedVolumeImpl(dataset, storageID, sd, fileIoProvider != null ? fileIoProvider : new FileIoProvider(null, null), conf); } if (null == usage) { // set usage unless overridden by unit tests usage = new DF(sd.getCurrentDir().getParentFile(), conf); } return new FsVolumeImpl( dataset, storageID, sd, fileIoProvider != null ? fileIoProvider : new FileIoProvider(null, null), conf, usage); } }
/** * Create a new file. * @throws IOException * if the file already exists or if the file cannot be created. */ public static File createFileWithExistsCheck( FsVolumeSpi volume, Block b, File f, FileIoProvider fileIoProvider) throws IOException { if (fileIoProvider.exists(volume, f)) { throw new IOException("Failed to create temporary file for " + b + ". File " + f + " should not be present, but is."); } // Create the zero-length temp file final boolean fileCreated; try { fileCreated = fileIoProvider.createFile(volume, f); } catch (IOException ioe) { throw new IOException(DISK_ERROR + "Failed to create " + f, ioe); } if (!fileCreated) { throw new IOException("Failed to create temporary file for " + b + ". File " + f + " should be creatable, but is already present."); } return f; }
File rbwDir = new File(bpCurrentDir, DataStorage.STORAGE_DIR_RBW); if (force) { fileIoProvider.fullyDelete(this, bpDir); } else { if (!fileIoProvider.delete(this, rbwDir)) { throw new IOException("Failed to delete " + rbwDir); !fileIoProvider.fullyDelete( this, finalizedDir)) { throw new IOException("Failed to delete " + finalizedDir); ((!DatanodeUtil.dirNoFilesRecursive( this, lazypersistDir, fileIoProvider) || !fileIoProvider.fullyDelete(this, lazypersistDir)))) { throw new IOException("Failed to delete " + lazypersistDir); fileIoProvider.fullyDelete(this, tmpDir); for (File f : fileIoProvider.listFiles(this, bpCurrentDir)) { if (!fileIoProvider.delete(this, f)) { throw new IOException("Failed to delete " + f); if (!fileIoProvider.delete(this, bpCurrentDir)) { throw new IOException("Failed to delete " + bpCurrentDir); for (File f : fileIoProvider.listFiles(this, bpDir)) { if (!fileIoProvider.delete(this, f)) { throw new IOException("Failed to delete " + f); if (!fileIoProvider.delete(this, bpDir)) {
/** * Recover an unlinked tmp file on datanode restart. If the original block * does not exist, then the tmp file is renamed to be the * original file name and the original name is returned; otherwise the tmp * file is deleted and null is returned. */ File recoverTempUnlinkedBlock(File unlinkedTmp) throws IOException { File blockFile = FsDatasetUtil.getOrigFile(unlinkedTmp); if (blockFile.exists()) { // If the original block file still exists, then no recovery is needed. if (!fileIoProvider.delete(volume, unlinkedTmp)) { throw new IOException("Unable to cleanup unlinked tmp file " + unlinkedTmp); } return null; } else { fileIoProvider.rename(volume, unlinkedTmp, blockFile); return blockFile; } }
boolean isBPDirEmpty(String bpid) throws IOException { File volumeCurrentDir = this.getCurrentDir(); File bpDir = new File(volumeCurrentDir, bpid); File bpCurrentDir = new File(bpDir, DataStorage.STORAGE_DIR_CURRENT); File finalizedDir = new File(bpCurrentDir, DataStorage.STORAGE_DIR_FINALIZED); File rbwDir = new File(bpCurrentDir, DataStorage.STORAGE_DIR_RBW); if (fileIoProvider.exists(this, finalizedDir) && !DatanodeUtil.dirNoFilesRecursive(this, finalizedDir, fileIoProvider)) { return false; } if (fileIoProvider.exists(this, rbwDir) && fileIoProvider.list(this, rbwDir).length != 0) { return false; } return true; }
File[] files = fileIoProvider.listFiles(volume, source); int numRecovered = 0; for (File file : files) { fileIoProvider.mkdirsWithExistsCheck(volume, targetDir); } catch(IOException ioe) { LOG.warn("Failed to mkdirs " + targetDir); fileIoProvider.rename(volume, metaFile, targetMetaFile); } catch (IOException e) { LOG.warn("Failed to move meta file from " fileIoProvider.rename(volume, blockFile, targetBlockFile); } catch (IOException e) { LOG.warn("Failed to move block file from " fileIoProvider.fullyDelete(volume, source); return numRecovered;
getVolume(), b, DatanodeUtil.getUnlinkTmpFile(file), fileIoProvider); try { try (FileInputStream in = fileIoProvider.getFileInputStream( getVolume(), file)) { try (FileOutputStream out = fileIoProvider.getFileOutputStream( getVolume(), tmpFile)) { IOUtils.copyBytes(in, out, 16 * 1024); + tmpFile.length()); fileIoProvider.replaceFile(getVolume(), tmpFile, file); } catch (IOException e) { if (!fileIoProvider.delete(getVolume(), tmpFile)) { DataNode.LOG.info("detachFile failed to delete temporary file " + tmpFile);
private void saveReplicas(BlockListAsLongs blocksListToPersist) { if (blocksListToPersist == null || blocksListToPersist.getNumberOfBlocks()== 0) { return; } final File tmpFile = new File(currentDir, REPLICA_CACHE_FILE + ".tmp"); final File replicaCacheFile = new File(currentDir, REPLICA_CACHE_FILE); if (!fileIoProvider.deleteWithExistsCheck(volume, tmpFile) || !fileIoProvider.deleteWithExistsCheck(volume, replicaCacheFile)) { return; } FileOutputStream out = null; try { out = fileIoProvider.getFileOutputStream(volume, tmpFile); blocksListToPersist.writeTo(out); out.close(); // Renaming the tmp file to replicas fileIoProvider.moveFile(volume, tmpFile, replicaCacheFile); } catch (Exception e) { // If write failed, the volume might be bad. Since the cache file is // not critical, log the error, delete both the files (tmp and cache) // and continue. LOG.warn("Failed to write replicas to cache ", e); fileIoProvider.deleteWithExistsCheck(volume, replicaCacheFile); } finally { IOUtils.closeStream(out); fileIoProvider.deleteWithExistsCheck(volume, tmpFile); } }
@Override public void save() throws IOException { state.lastSavedMs = Time.now(); boolean success = false; try (BufferedWriter writer = new BufferedWriter( new OutputStreamWriter(fileIoProvider.getFileOutputStream( FsVolumeImpl.this, getTempSaveFile()), "UTF-8"))) { WRITER.writeValue(writer, state); success = true; } finally { if (!success) { fileIoProvider.delete(FsVolumeImpl.this, getTempSaveFile()); } } fileIoProvider.move(FsVolumeImpl.this, getTempSaveFile().toPath(), getSaveFile().toPath(), StandardCopyOption.ATOMIC_MOVE); if (LOG.isTraceEnabled()) { LOG.trace("save({}, {}): saved {}", storageID, bpid, WRITER.writeValueAsString(state)); } }
final FileInputStream fis = fileIoProvider.getFileInputStream( volume, metaFile); DataChecksum dcs = BlockMetadataHeader.readHeader(fis).getChecksum(); byte[] b = new byte[Math.max(lastchunksize, checksumsize)]; try (RandomAccessFile blockRAF = fileIoProvider.getRandomAccessFile( volume, blockFile, "rw")) { try (RandomAccessFile metaRAF = fileIoProvider.getRandomAccessFile( volume, metaFile, "rw")) { metaRAF.setLength(newmetalen);
@Override public OutputStream createRestartMetaStream() throws IOException { File blockFile = getBlockFile(); File restartMeta = new File(blockFile.getParent() + File.pathSeparator + "." + blockFile.getName() + ".restart"); if (!getFileIoProvider().deleteWithExistsCheck(getVolume(), restartMeta)) { DataNode.LOG.warn("Failed to delete restart meta file: " + restartMeta.getPath()); } return getFileIoProvider().getFileOutputStream(getVolume(), restartMeta); }
inputStream = fileIoProvider.getFileInputStream(volume, replicaFile); BlockListAsLongs blocksList = BlockListAsLongs.readFrom(inputStream, maxDataLength); if (!fileIoProvider.delete(volume, replicaFile)) { LOG.info("Failed to delete replica cache file: " + replicaFile.getPath());
fileIoProvider.fullyDelete(volume, tmpDir); fileIoProvider.mkdirs(volume, rbwDir); fileIoProvider.mkdirs(volume, tmpDir);
getFileIoProvider().getRandomAccessFile(getVolume(), metaFile, "rw"); FileOutputStream crcOut = null; try { blockOut = fileIoProvider.getFileOutputStream( getVolume(), new RandomAccessFile(blockFile, "rw").getFD()); crcOut = fileIoProvider.getFileOutputStream(getVolume(), metaRAF.getFD()); if (!isCreate) { blockOut.getChannel().position(blockDiskSize);
fileIoProvider.exists(vol, diskMetaFile); final boolean diskFileExists = diskFile != null && fileIoProvider.exists(vol, diskFile); if (diskMetaFileExists && fileIoProvider.delete(vol, diskMetaFile)) { LOG.warn("Deleted a metadata file without a block " + diskMetaFile.getAbsolutePath()); + " from memory with missing block file on the disk"); if (diskMetaFileExists && fileIoProvider.delete(vol, diskMetaFile)) { LOG.warn("Deleted a metadata file for the deleted block " + diskMetaFile.getAbsolutePath()); if (!fileIoProvider.delete(vol, diskFile)) { LOG.warn("Failed to delete " + diskFile); File memMetaFile = FsDatasetUtil.getMetaFile(diskFile, memBlockInfo.getGenerationStamp()); if (fileIoProvider.exists(vol, memMetaFile)) { String warningPrefix = "Metadata file in memory " + memMetaFile.getAbsolutePath()
/** * Flush the data stream if it supports it. */ public void flushDataOut() throws IOException { if (dataOut != null) { fileIoProvider.flush(volume, dataOut); } }
/** * Create a FileOutputStream using * {@link FileOutputStream#FileOutputStream(File, boolean)}. * * Wraps the created output stream to intercept write calls * before delegating to the wrapped stream. * * @param volume target volume. null if unavailable. * @param f File object. * @return FileOutputStream to the given file object. * @throws FileNotFoundException */ public FileOutputStream getFileOutputStream( @Nullable FsVolumeSpi volume, File f) throws FileNotFoundException { return getFileOutputStream(volume, f, false); }
@Override public boolean deleteMetadata() { return getFileIoProvider().fullyDelete(getVolume(), getMetaFile()); }
@Override public boolean blockDataExists() { return getFileIoProvider().exists(getVolume(), getBlockFile()); }
/** * Write the current dfsUsed to the cache file. */ void saveDfsUsed() { File outFile = new File(currentDir, DU_CACHE_FILE); if (!fileIoProvider.deleteWithExistsCheck(volume, outFile)) { FsDatasetImpl.LOG.warn("Failed to delete old dfsUsed file in " + outFile.getParent()); } try { long used = getDfsUsed(); try (Writer out = new OutputStreamWriter( new FileOutputStream(outFile), "UTF-8")) { // mtime is written last, so that truncated writes won't be valid. out.write(Long.toString(used) + " " + Long.toString(timer.now())); // This is only called as part of the volume shutdown. // We explicitly avoid calling flush with fileIoProvider which triggers // volume check upon io exception to avoid cyclic volume checks. out.flush(); } } catch (IOException ioe) { // If write failed, the volume might be bad. Since the cache file is // not critical, log the error and continue. FsDatasetImpl.LOG.warn("Failed to write dfsUsed to " + outFile, ioe); } }