for (SegmentDescriptor s : segmentsToMerge) { approxOutputSize += s.segmentLength + ChecksumFileSystem.getApproxChkSumLength( s.segmentLength);
/** Return the length of the checksum file given the size of the * actual file. **/ public long getChecksumFileLength(Path file, long fileSize) { return getChecksumLength(fileSize, getBytesPerSum()); }
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize) throws IOException { super( file, fs.getFileStatus(file).getReplication() ); this.datas = fs.getRawFileSystem().open(file, bufferSize); this.fs = fs; Path sumFile = fs.getChecksumFile(file); try { int sumBufferSize = fs.getSumBufferSize(fs.getBytesPerSum(), bufferSize); sums = fs.getRawFileSystem().open(sumFile, sumBufferSize); byte[] version = new byte[CHECKSUM_VERSION.length]; sums.readFully(version); if (!Arrays.equals(version, CHECKSUM_VERSION)) throw new IOException("Not a checksum file: "+sumFile); this.bytesPerSum = sums.readInt(); set(fs.verifyChecksum, DataChecksum.newCrc32(), bytesPerSum, 4); } catch (IOException e) { // mincing the message is terrible, but java throws permission // exceptions as FNF because that's all the method signatures allow! if (!(e instanceof FileNotFoundException) || e.getMessage().endsWith(" (Permission denied)")) { LOG.warn("Problem opening checksum file: "+ file + ". Ignoring exception: " , e); } set(fs.verifyChecksum, null, 1, 0); } }
public ChecksumFSOutputSummer(ChecksumFileSystem fs, Path file, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress, FsPermission permission) throws IOException { super(DataChecksum.newDataChecksum(DataChecksum.Type.CRC32, fs.getBytesPerSum())); int bytesPerSum = fs.getBytesPerSum(); this.datas = fs.getRawFileSystem().create(file, permission, overwrite, bufferSize, replication, blockSize, progress); int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize); this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), permission, true, sumBufferSize, replication, blockSize, null); sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length); sums.writeInt(bytesPerSum); }
if (!fs.isDirectory(src)) { // source is a file fs.copyToLocalFile(src, dst); FileSystem localFs = getLocal(getConf()).getRawFileSystem(); if (localFs.isDirectory(dst)) { dst = new Path(dst, src.getName()); dst = getChecksumFile(dst); if (localFs.exists(dst)) { //remove old local checksum file localFs.delete(dst, true); Path checksumFile = getChecksumFile(src); if (copyCrc && fs.exists(checksumFile)) { //copy checksum file fs.copyToLocalFile(checksumFile, dst); FileStatus[] srcs = listStatus(src); for (FileStatus srcFile : srcs) { copyToLocalFile(srcFile.getPath(), new Path(dst, srcFile.getPath().getName()), copyCrc);
Path parent = f.getParent(); if (parent != null) { if (!createParent && !exists(parent)) { throw new FileNotFoundException("Parent directory doesn't exist: " + parent); } else if (!mkdirs(parent)) { throw new IOException("Mkdirs failed to create " + parent + " (exists=" + exists(parent) + ", cwd=" + getWorkingDirectory() + ")"); blockSize, progress); Path checkFile = getChecksumFile(f); if (fs.exists(checkFile)) { fs.delete(checkFile, true);
/** * Opens an FSDataInputStream at the indicated Path. * @param f the file name to open * @param bufferSize the size of the buffer to be used. */ @Override public FSDataInputStream open(Path f, int bufferSize) throws IOException { FileSystem fs; InputStream in; if (verifyChecksum) { fs = this; in = new ChecksumFSInputChecker(this, f, bufferSize); } else { fs = getRawFileSystem(); in = fs.open(f, bufferSize); } return new FSDataBoundedInputStream(fs, f, in); }
public enum LsOption { Recursive, WithBlockSize };
/** * Implement the delete(Path, boolean) in checksum * file system. */ @Override public boolean delete(Path f, boolean recursive) throws IOException{ FileStatus fstatus = null; try { fstatus = fs.getFileStatus(f); } catch(FileNotFoundException e) { return false; } if (fstatus.isDirectory()) { //this works since the crcs are in the same //directories and the files. so we just delete //everything in the underlying filesystem return fs.delete(f, recursive); } else { Path checkFile = getChecksumFile(f); if (fs.exists(checkFile)) { fs.delete(checkFile, true); } return fs.delete(f, true); } }
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file) throws IOException { this(fs, file, fs.getConf().getInt( LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT)); }
public void testgetChecksumLength() throws Exception { assertEquals(8, ChecksumFileSystem.getChecksumLength(0L, 512)); assertEquals(12, ChecksumFileSystem.getChecksumLength(1L, 512)); assertEquals(12, ChecksumFileSystem.getChecksumLength(512L, 512)); assertEquals(16, ChecksumFileSystem.getChecksumLength(513L, 512)); assertEquals(16, ChecksumFileSystem.getChecksumLength(1023L, 512)); assertEquals(16, ChecksumFileSystem.getChecksumLength(1024L, 512)); assertEquals(408, ChecksumFileSystem.getChecksumLength(100L, 1)); assertEquals(4000000000008L, ChecksumFileSystem.getChecksumLength(10000000000000L, 10)); }
/** * Set replication for an existing file. * Implement the abstract <tt>setReplication</tt> of <tt>FileSystem</tt> * @param src file name * @param replication new replication * @throws IOException * @return true if successful; * false if file does not exist or is a directory */ public boolean setReplication(Path src, short replication) throws IOException { boolean value = fs.setReplication(src, replication); if (!value) return false; Path checkFile = getChecksumFile(src); if (exists(checkFile)) fs.setReplication(checkFile, replication); return true; }
private FSDataOutputStream create(Path f, FsPermission permission, boolean overwrite, boolean createParent, int bufferSize, short replication, long blockSize, int bytesPerChecksum, Progressable progress) throws IOException { Path parent = f.getParent(); if (parent != null) { if (!createParent && !exists(parent)) { throw new FileNotFoundException("Parent directory doesn't exist: " + parent); } else if (!mkdirs(parent)) { throw new IOException("Mkdirs failed to create " + parent); } } final FSDataOutputStream out = new FSDataOutputStream( new ChecksumFSOutputSummer(this, f, overwrite, bufferSize, replication, blockSize, bytesPerChecksum, progress), null); if (permission != null) { setPermission(f, permission); } return out; }
/** {@inheritDoc} */ @Override public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return create(f, permission, overwrite, false, bufferSize, replication, blockSize, getConf().getInt("io.bytes.per.checksum", 512), progress); }
if (!fs.isDirectory(src)) { // source is a file fs.copyToLocalFile(src, dst); FileSystem localFs = getLocal(getConf()); if (localFs instanceof ChecksumFileSystem) { localFs = ((ChecksumFileSystem) localFs).getRawFileSystem(); dst = getChecksumFile(dst); if (localFs.exists(dst)) { //remove old local checksum file localFs.delete(dst, true); Path checksumFile = getChecksumFile(src); if (copyCrc && fs.exists(checksumFile)) { //copy checksum file fs.copyToLocalFile(checksumFile, dst); FileStatus[] srcs = listStatus(src); for (FileStatus srcFile : srcs) { copyToLocalFile(srcFile.getPath(), new Path(dst, srcFile.getPath().getName()), copyCrc);
public ChecksumFSOutputSummer(ChecksumFileSystem fs, Path file, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { super(new CRC32(), fs.getBytesPerSum(), 4); int bytesPerSum = fs.getBytesPerSum(); this.datas = fs.getRawFileSystem().create(file, overwrite, bufferSize, replication, blockSize, progress); int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize); this.sums = fs.getRawFileSystem().create(fs.getChecksumFile(file), true, sumBufferSize, replication, blockSize); sums.write(CHECKSUM_VERSION, 0, CHECKSUM_VERSION.length); sums.writeInt(bytesPerSum); }
Path parent = f.getParent(); if (parent != null) { if (!createParent && !exists(parent)) { throw new FileNotFoundException("Parent directory doesn't exist: " + parent); } else if (!mkdirs(parent)) { throw new IOException("Mkdirs failed to create " + parent + " (exists=" + exists(parent) + ", cwd=" + getWorkingDirectory() + ")"); blockSize, progress); Path checkFile = getChecksumFile(f); if (fs.exists(checkFile)) { fs.delete(checkFile, true);
/** * The src file is under FS, and the dst is on the local disk. * Copy it from FS control to the local dst name. */ @Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { Configuration conf = getConf(); FileUtil.copy(this, src, getLocal(conf), dst, delSrc, conf); }
private FSDataOutputStream create(String filename, boolean noChecksum) throws IOException { Path filePath = qualifiedPath(filename); // even though it was qualified using the default FS, it may not be in it FileSystem fs = filePath.getFileSystem(getConf()); if (noChecksum && fs instanceof ChecksumFileSystem) { fs = ((ChecksumFileSystem) fs).getRawFileSystem(); } return fs.create(filePath, true /* overwrite */); }