@Override protected void post(final Path file, final MessageDigest digest, final BaseB2Response response) throws BackgroundException { this.verify(file, digest, Checksum.parse(StringUtils.removeStart(((B2FileResponse) response).getContentSha1(), "unverified:"))); }
@Override protected void verify(final Path file, final MessageDigest digest, final Checksum checksum) throws ChecksumException { if(null == digest) { log.debug(String.format("Digest verification disabled for file %s", file)); return; } if(file.getType().contains(Path.Type.encrypted)) { log.warn(String.format("Skip checksum verification for %s with client side encryption enabled", file)); return; } final String expected = Hex.encodeHexString(digest.digest()); if(!checksum.equals(Checksum.parse(expected))) { throw new ChecksumException(MessageFormat.format(LocaleFactory.localizedString("Upload {0} failed", "Error"), file.getName()), MessageFormat.format("Mismatch between {0} hash {1} of uploaded data and ETag {2} returned by the server", checksum.algorithm.toString(), expected, checksum.hash)); } }
public static Checksum parse(final String hash) { if(StringUtils.isBlank(hash)) { return Checksum.NONE; } if(hash.matches("[a-fA-F0-9]{32}")) { return new Checksum(HashAlgorithm.md5, hash); } if(hash.matches("[a-fA-F0-9]{40}")) { return new Checksum(HashAlgorithm.sha1, hash); } if(hash.matches("[A-Fa-f0-9]{64}")) { return new Checksum(HashAlgorithm.sha256, hash); } if(hash.matches("[A-Fa-f0-9]{128}")) { return new Checksum(HashAlgorithm.sha512, hash); } if(hash.matches("[a-fA-F0-9]{8}")) { return new Checksum(HashAlgorithm.crc32, hash); } log.warn(String.format("Failure to detect algorithm for checksum %s", hash)); return Checksum.NONE; }
@Override public Comparison compare(final Attributes remote, final Attributes local) { if(Checksum.NONE == remote.getChecksum()) { log.warn(String.format("No remote checksum available for comparison %s", remote)); return Comparison.unknown; } if(Checksum.NONE == local.getChecksum()) { log.warn(String.format("No local checksum available for comparison %s", local)); return Comparison.unknown; } if(remote.getChecksum().equals(local.getChecksum())) { return Comparison.equal; } return Comparison.notequal; } }
final Checksum previous = new Checksum(HashAlgorithm.md5, preferences.getProperty(String.format("%s.checksum", this.getConfiguration()))); if(log.isDebugEnabled()) { if(previous.equals(current)) { if(log.isInfoEnabled()) { log.info(String.format("Skip importing bookmarks from %s with previously saved checksum %s", file, previous));
@Override public int hashCode() { int result = (int) (size ^ (size >>> 32)); result = 31 * result + (int) (modified ^ (modified >>> 32)); result = 31 * result + (permission != null ? permission.hashCode() : 0); result = 31 * result + (checksum != null ? checksum.hashCode() : 0); result = 31 * result + (versionId != null ? versionId.hashCode() : 0); result = 31 * result + (revision != null ? revision.hashCode() : 0); result = 31 * result + (region != null ? region.hashCode() : 0); return result; }
return Worker.empty(); if(current.equals(checksum)) { if(log.isInfoEnabled()) { log.info(String.format("File %s not modified with checksum %s", local, current));
@Override protected void post(final Path file, final MessageDigest digest, final StorageObject part) throws BackgroundException { if(null != part.getServerSideEncryptionAlgorithm()) { log.warn(String.format("Skip checksum verification for %s with server side encryption enabled", file)); return; } this.verify(file, digest, Checksum.parse(part.getETag())); }
protected void verify(final Path file, final MessageDigest digest, final Checksum checksum) throws ChecksumException { if(file.getType().contains(Path.Type.encrypted)) { log.warn(String.format("Skip checksum verification for %s with client side encryption enabled", file)); return; } if(null == digest) { log.debug(String.format("Digest disabled for file %s", file)); return; } if(null == checksum) { log.warn("ETag returned by server is unknown checksum algorithm"); return; } if(!checksum.algorithm.equals(HashAlgorithm.md5)) { log.warn(String.format("ETag %s returned by server is %s but expected MD5", checksum.hash, checksum.algorithm)); return; } // Obtain locally-calculated MD5 hash. final String expected = Hex.encodeHexString(digest.digest()); // Compare our locally-calculated hash with the ETag returned by S3. if(!checksum.equals(Checksum.parse(expected))) { throw new ChecksumException(MessageFormat.format(LocaleFactory.localizedString("Upload {0} failed", "Error"), file.getName()), MessageFormat.format("Mismatch between MD5 hash {0} of uploaded data and ETag {1} returned by the server", expected, checksum.hash)); } }
if(Checksum.NONE != append.checksum) { final ChecksumCompute compute = ChecksumComputeFactory.get(append.checksum.algorithm); if(compute.compute(local.getInputStream(), parent).equals(append.checksum)) { if(log.isInfoEnabled()) { log.info(String.format("Skip file %s with checksum %s", file, local.attributes().getChecksum()));
@Override public Checksum compute(final InputStream in, final TransferStatus status) throws ChecksumException { return new Checksum(HashAlgorithm.sha256, Hex.encodeHexString(this.digest("SHA-256", this.normalize(in, status)))); } }
@Override protected void post(final Path file, final MessageDigest digest, final StorageObject response) throws BackgroundException { this.verify(file, digest, Checksum.parse(response.getMd5sum())); } }
.getDataObjectChecksumUtilitiesAO(fs.getIRODSAccount()); final ChecksumValue value = checksum.computeChecksumOnDataObject(f); final Checksum fingerprint = Checksum.parse(value.getChecksumStringValue()); if(null == fingerprint) { log.warn(String.format("Unsupported checksum algorithm %s", value.getChecksumEncoding())); if(!expected.equals(fingerprint)) { throw new ChecksumException(MessageFormat.format(LocaleFactory.localizedString("Upload {0} failed", "Error"), file.getName()), MessageFormat.format("Mismatch between {0} hash {1} of uploaded data and ETag {2} returned by the server",
if(Checksum.NONE != attributes.getChecksum()) { final ChecksumCompute compute = ChecksumComputeFactory.get(attributes.getChecksum().algorithm); if(compute.compute(local.getInputStream(), parent).equals(attributes.getChecksum())) { if(log.isInfoEnabled()) { log.info(String.format("Skip file %s with checksum %s", file, local.attributes().getChecksum()));
@Override public Checksum compute(final InputStream in, final TransferStatus status) throws ChecksumException { return new Checksum(HashAlgorithm.sha1, Hex.encodeHexString(this.digest("SHA-1", this.normalize(in, status)))); } }
protected PathAttributes toAttributes(final B2FileResponse response) { final PathAttributes attributes = new PathAttributes(); attributes.setSize(response.getContentLength()); if(response.getFileInfo().containsKey(X_BZ_INFO_LARGE_FILE_SHA1)) { attributes.setChecksum(Checksum.parse(response.getFileInfo().get(X_BZ_INFO_LARGE_FILE_SHA1))); } else { attributes.setChecksum(Checksum.parse(StringUtils.removeStart(StringUtils.lowerCase(response.getContentSha1(), Locale.ROOT), "unverified:"))); } final Map<String, String> metadata = new HashMap<>(); for(Map.Entry<String, String> entry : response.getFileInfo().entrySet()) { metadata.put(entry.getKey(), entry.getValue()); } attributes.setMetadata(metadata); attributes.setVersionId(response.getFileId()); if(response.getFileInfo().containsKey(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS)) { attributes.setModificationDate(Long.valueOf(response.getFileInfo().get(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS))); } return attributes; }
file.getName())); final Checksum download = compute.compute(local.getInputStream(), new TransferStatus()); if(!checksum.equals(download)) { throw new ChecksumException( MessageFormat.format(LocaleFactory.localizedString("Download {0} failed", "Error"), file.getName()),
@Override public Checksum compute(final InputStream in, final TransferStatus status) throws ChecksumException { return new Checksum(HashAlgorithm.sha512, Hex.encodeHexString(this.digest("SHA-512", this.normalize(in, status)))); } }
protected PathAttributes toAttributes(final StorageObject object) { final PathAttributes attributes = new PathAttributes(); if(StringUtils.isNotBlank(object.getMd5sum())) { // For manifest files, the ETag in the response for a GET or HEAD on the manifest file is the MD5 sum of // the concatenated string of ETags for each of the segments in the manifest. attributes.setChecksum(Checksum.parse(object.getMd5sum())); } attributes.setSize(object.getSize()); final String lastModified = object.getLastModified(); if(lastModified != null) { try { attributes.setModificationDate(iso8601DateParser.parse(lastModified).getTime()); } catch(InvalidDateException e) { log.warn(String.format("%s is not ISO 8601 format %s", lastModified, e.getMessage())); try { attributes.setModificationDate(rfc1123DateFormatter.parse(lastModified).getTime()); } catch(InvalidDateException f) { log.warn(String.format("%s is not RFC 1123 format %s", lastModified, f.getMessage())); } } } return attributes; }
@Override public Checksum compute(final InputStream in, final TransferStatus status) throws ChecksumException { return new Checksum(HashAlgorithm.md5, Hex.encodeHexString(this.digest("MD5", this.normalize(in, status)))); }