/** * Get the checksum of a file, if the FS supports checksums. * * @param f The file path * @return The file checksum. The default return value is null, * which indicates that no checksum algorithm is implemented * in the corresponding FileSystem. * @throws IOException IO failure */ public FileChecksum getFileChecksum(Path f) throws IOException { return getFileChecksum(f, Long.MAX_VALUE); }
@Override public FileChecksum getFileChecksum(Path f, long length) throws IOException { return fs.getFileChecksum(f, length); }
@Override public FileChecksum getFileChecksum(Path f) throws IOException { return fs.getFileChecksum(f); }
private FileChecksum getFileChecksum(final FileSystem fs, final Path path) { try { return fs.getFileChecksum(path); } catch (IOException e) { LOG.warn("Unable to get checksum for file=" + path, e); return null; } }
@Override public FileChecksum getFileChecksum(Path f) throws IOException { checkPath(f); return fsImpl.getFileChecksum(f); }
public FileChecksum getFileChecksum(Path f) throws java.io.IOException { return this.underlyingFs.getFileChecksum(replaceScheme(f, this.replacementScheme, this.underlyingScheme)); }
@Override // For pfile, calculate the checksum for use in testing public FileChecksum getFileChecksum(Path f) throws IOException { if (scheme.equalsIgnoreCase("pfile") && fs.isFile(f)) { return getPFileChecksum(f); } return fs.getFileChecksum(f); }
@Override public FileChecksum getFileChecksum(final Path f, final long length) throws AccessControlException, FileNotFoundException, IOException { InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(f), true); return res.targetFileSystem.getFileChecksum(res.remainingPath, length); }
@Override public FileChecksum getFileChecksum(final Path f) throws AccessControlException, FileNotFoundException, IOException { InodeTree.ResolveResult<FileSystem> res = fsState.resolve(getUriPath(f), true); return res.targetFileSystem.getFileChecksum(res.remainingPath); }
static public String checksumFor(Path path, FileSystem fs) throws IOException { // TODO: fs checksum only available on hdfs, need to // find a solution for other fs (eg, local fs, s3, etc) String checksumString = null; FileChecksum checksum = fs.getFileChecksum(path); if (checksum != null) { checksumString = StringUtils.byteToHexString( checksum.getBytes(), 0, checksum.getLength()); } return checksumString; }
@Override protected void processPath(PathData item) throws IOException { if (item.stat.isDirectory()) { throw new PathIsDirectoryException(item.toString()); } FileChecksum checksum = item.fs.getFileChecksum(item.path); if (checksum == null) { out.printf("%s\tNONE\t%n", item.toString()); } else { String checksumString = StringUtils.byteToHexString( checksum.getBytes(), 0, checksum.getLength()); out.printf("%s\t%s\t%s%n", item.toString(), checksum.getAlgorithmName(), checksumString); } } }
private static void addInsertNonDirectoryInformation(Path p, FileSystem fileSystem, InsertEventRequestData insertData) throws IOException { insertData.addToFilesAdded(p.toString()); FileChecksum cksum = fileSystem.getFileChecksum(p); String acidDirPath = AcidUtils.getFirstLevelAcidDirPath(p.getParent(), fileSystem); // File checksum is not implemented for local filesystem (RawLocalFileSystem) if (cksum != null) { String checksumString = StringUtils.byteToHexString(cksum.getBytes(), 0, cksum.getLength()); insertData.addToFilesAddedChecksum(checksumString); } else { // Add an empty checksum string for filesystems that don't generate one insertData.addToFilesAddedChecksum(""); } // acid dir will be present only for acid write operations. if (acidDirPath != null) { insertData.addToSubDirectoryList(acidDirPath); } }
FileChecksum checksumTmp = this.origin.isDirectory() ? null : this.originFs.getFileChecksum(this.origin.getPath()); this.checksum = checksumTmp == null ? new byte[0] : checksumTmp.getBytes();
for (Path p : newFiles) { insertData.addToFilesAdded(p.toString()); FileChecksum cksum = fileSystem.getFileChecksum(p);
@Override public FileChecksum getFileChecksum(final Path f, final long length) throws IOException { try { return underlyingFs.getFileChecksum(f, length); } catch(FSError e) { throw propagateFSError(e); } }
@Override public FileChecksum getFileChecksum(Path f) throws IOException { try { return underlyingFs.getFileChecksum(f); } catch(FSError e) { throw propagateFSError(e); } }
@Override public FileChecksum getFileChecksum(Path f) throws AccessControlException, FileNotFoundException, IOException { return fs.getFileChecksum(f); }
private static SortedMap<String, String> getClusterChecksumMap() throws IOException { FileSystem fileSystem = FileSystem.get(hConf); List<String> fileList = addAllFiles(fileSystem); SortedMap<String, String> checksumMap = new TreeMap<String, String>(); for (String file : fileList) { FileChecksum fileChecksum = fileSystem.getFileChecksum(new Path(file)); checksumMap.put(normalizedFileName(file), fileChecksum.toString()); } LOG.info("Added " + checksumMap.size() + " checksums for snapshot files."); return checksumMap; }
@Override // For pfile, calculate the checksum for use in testing public FileChecksum getFileChecksum(Path f) throws IOException { if (scheme.equalsIgnoreCase("pfile") && fs.isFile(f)) { return getPFileChecksum(f); } return fs.getFileChecksum(f); }
@Test public void testLongLivedReadClientAfterRestart() throws IOException { FileChecksum checksum = writeUnencryptedAndThenRestartEncryptedCluster(); assertEquals(PLAIN_TEXT, DFSTestUtil.readFile(fs, TEST_PATH)); assertEquals(checksum, fs.getFileChecksum(TEST_PATH)); // Restart the NN and DN, after which the client's encryption key will no // longer be valid. cluster.restartNameNode(); assertTrue(cluster.restartDataNode(0)); assertEquals(PLAIN_TEXT, DFSTestUtil.readFile(fs, TEST_PATH)); assertEquals(checksum, fs.getFileChecksum(TEST_PATH)); }