summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRoss Burton <ross.burton@intel.com>2018-08-13 18:02:25 (GMT)
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-08-29 14:23:58 (GMT)
commit1b966ffd5fcfdc8e0fd1ed622b71016e732ee7e6 (patch)
tree5d0803668dad7d6af7e2b4e63527f43c1d55a991
parentf7a8c088ae1db6c2ebaa5e0aa4c7c8ed352bb045 (diff)
downloadpoky-1b966ffd5fcfdc8e0fd1ed622b71016e732ee7e6.tar.gz
bitbake: utils/md5_file: don't iterate line-by-line
Opening a file in binary mode and iterating it seems like the simple solution but will still break on newlines, which for binary files isn't really useful as the size of the chunks could be huge or tiny. Instead, let's be a bit more clever: we'll be MD5ing lots of files, but we don't want to fill up memory: use mmap() to open the file and read the file in 8k blocks. (Bitbake rev: f3f28ec7e6cad5003014462feeb9178496107e08) Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/utils.py13
1 files changed, 9 insertions, 4 deletions
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index c540b49..f753123 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -523,12 +523,17 @@ def md5_file(filename):
523 """ 523 """
524 Return the hex string representation of the MD5 checksum of filename. 524 Return the hex string representation of the MD5 checksum of filename.
525 """ 525 """
526 import hashlib 526 import hashlib, mmap
527 m = hashlib.md5()
528 527
529 with open(filename, "rb") as f: 528 with open(filename, "rb") as f:
530 for line in f: 529 m = hashlib.md5()
531 m.update(line) 530 try:
531 with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
532 for chunk in iter(lambda: mm.read(8192), b''):
533 m.update(chunk)
534 except ValueError:
535 # You can't mmap() an empty file so silence this exception
536 pass
532 return m.hexdigest() 537 return m.hexdigest()
533 538
534def sha256_file(filename): 539def sha256_file(filename):