summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/utils.py
diff options
context:
space:
mode:
authorRoss Burton <ross.burton@intel.com>2019-11-07 23:57:33 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-11-25 21:26:15 +0000
commit0d6c922af9c2d6f5e6f3f94b4c6875f2a4854646 (patch)
tree63420ea66a87fa640f6025b2e2ae73722b44f2f5 /bitbake/lib/bb/utils.py
parent59edce3af55111c23a9189eaaff6b345774cb11b (diff)
downloadpoky-0d6c922af9c2d6f5e6f3f94b4c6875f2a4854646.tar.gz
bitbake: utils: also use mmap for SHA256 and SHA1, for performance
md5_file() uses a mmap() window to improve performance when hashing files, so refactor the code and do the same for SHA1 and SHA256. (Bitbake rev: 94ede642dce8cdbf09f566e3f7e9e260d33fda27) Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> Signed-off-by: Armin Kuster <akuster808@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/utils.py')
-rw-r--r--bitbake/lib/bb/utils.py34
1 files changed, 14 insertions, 20 deletions
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index d035949b3d..8d40bcdf83 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -520,22 +520,26 @@ def unlockfile(lf):
520 fcntl.flock(lf.fileno(), fcntl.LOCK_UN) 520 fcntl.flock(lf.fileno(), fcntl.LOCK_UN)
521 lf.close() 521 lf.close()
522 522
523def md5_file(filename): 523def _hasher(method, filename):
524 """ 524 import mmap
525 Return the hex string representation of the MD5 checksum of filename.
526 """
527 import hashlib, mmap
528 525
529 with open(filename, "rb") as f: 526 with open(filename, "rb") as f:
530 m = hashlib.md5()
531 try: 527 try:
532 with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm: 528 with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
533 for chunk in iter(lambda: mm.read(8192), b''): 529 for chunk in iter(lambda: mm.read(8192), b''):
534 m.update(chunk) 530 method.update(chunk)
535 except ValueError: 531 except ValueError:
536 # You can't mmap() an empty file so silence this exception 532 # You can't mmap() an empty file so silence this exception
537 pass 533 pass
538 return m.hexdigest() 534 return method.hexdigest()
535
536
537def md5_file(filename):
538 """
539 Return the hex string representation of the MD5 checksum of filename.
540 """
541 import hashlib
542 return _hasher(hashlib.md5(), filename)
539 543
540def sha256_file(filename): 544def sha256_file(filename):
541 """ 545 """
@@ -543,24 +547,14 @@ def sha256_file(filename):
543 filename. 547 filename.
544 """ 548 """
545 import hashlib 549 import hashlib
546 550 return _hasher(hashlib.sha256(), filename)
547 s = hashlib.sha256()
548 with open(filename, "rb") as f:
549 for line in f:
550 s.update(line)
551 return s.hexdigest()
552 551
553def sha1_file(filename): 552def sha1_file(filename):
554 """ 553 """
555 Return the hex string representation of the SHA1 checksum of the filename 554 Return the hex string representation of the SHA1 checksum of the filename
556 """ 555 """
557 import hashlib 556 import hashlib
558 557 return _hasher(hashlib.sha1(), filename)
559 s = hashlib.sha1()
560 with open(filename, "rb") as f:
561 for line in f:
562 s.update(line)
563 return s.hexdigest()
564 558
565def preserved_envvars_exported(): 559def preserved_envvars_exported():
566 """Variables which are taken from the environment and placed in and exported 560 """Variables which are taken from the environment and placed in and exported