From 0d6c922af9c2d6f5e6f3f94b4c6875f2a4854646 Mon Sep 17 00:00:00 2001 From: Ross Burton Date: Thu, 7 Nov 2019 23:57:33 +0000 Subject: bitbake: utils: also use mmap for SHA256 and SHA1, for performance md5_file() uses a mmap() window to improve performance when hashing files, so refactor the code and do the same for SHA1 and SHA256. (Bitbake rev: 94ede642dce8cdbf09f566e3f7e9e260d33fda27) Signed-off-by: Ross Burton Signed-off-by: Richard Purdie Signed-off-by: Armin Kuster Signed-off-by: Richard Purdie --- bitbake/lib/bb/utils.py | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) (limited to 'bitbake/lib/bb/utils.py') diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py index d035949b3d..8d40bcdf83 100644 --- a/bitbake/lib/bb/utils.py +++ b/bitbake/lib/bb/utils.py @@ -520,22 +520,26 @@ def unlockfile(lf): fcntl.flock(lf.fileno(), fcntl.LOCK_UN) lf.close() -def md5_file(filename): - """ - Return the hex string representation of the MD5 checksum of filename. - """ - import hashlib, mmap +def _hasher(method, filename): + import mmap with open(filename, "rb") as f: - m = hashlib.md5() try: with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm: for chunk in iter(lambda: mm.read(8192), b''): - m.update(chunk) + method.update(chunk) except ValueError: # You can't mmap() an empty file so silence this exception pass - return m.hexdigest() + return method.hexdigest() + + +def md5_file(filename): + """ + Return the hex string representation of the MD5 checksum of filename. + """ + import hashlib + return _hasher(hashlib.md5(), filename) def sha256_file(filename): """ @@ -543,24 +547,14 @@ def sha256_file(filename): filename. """ import hashlib - - s = hashlib.sha256() - with open(filename, "rb") as f: - for line in f: - s.update(line) - return s.hexdigest() + return _hasher(hashlib.sha256(), filename) def sha1_file(filename): """ Return the hex string representation of the SHA1 checksum of the filename """ import hashlib - - s = hashlib.sha1() - with open(filename, "rb") as f: - for line in f: - s.update(line) - return s.hexdigest() + return _hasher(hashlib.sha1(), filename) def preserved_envvars_exported(): """Variables which are taken from the environment and placed in and exported -- cgit v1.2.3-54-g00ecf