summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorRoss Burton <ross.burton@intel.com>2018-08-13 19:02:25 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-08-15 09:45:02 +0100
commitbeb93fff65c1aa677d341c4531bf15ce46f04a69 (patch)
treeb2edf7f9f649868a44d4806f39574d3afa836630 /bitbake
parent956278f4ec998ea665fc9ebe45a188bded919179 (diff)
downloadpoky-beb93fff65c1aa677d341c4531bf15ce46f04a69.tar.gz
bitbake: utils/md5_file: don't iterate line-by-line
Opening a file in binary mode and iterating it seems like the simple solution but will still break on newlines, which for binary files isn't really useful as the size of the chunks could be huge or tiny. Instead, let's be a bit more clever: we'll be MD5ing lots of files, but we don't want to fill up memory: use mmap() to open the file and read the file in 8k blocks. (Bitbake rev: a0ac8d67f1471a0c611d691b856fede67efb53f6) Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/utils.py13
1 files changed, 9 insertions, 4 deletions
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index 9903183213..b20cdabcf0 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -524,12 +524,17 @@ def md5_file(filename):
524 """ 524 """
525 Return the hex string representation of the MD5 checksum of filename. 525 Return the hex string representation of the MD5 checksum of filename.
526 """ 526 """
527 import hashlib 527 import hashlib, mmap
528 m = hashlib.md5()
529 528
530 with open(filename, "rb") as f: 529 with open(filename, "rb") as f:
531 for line in f: 530 m = hashlib.md5()
532 m.update(line) 531 try:
532 with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mm:
533 for chunk in iter(lambda: mm.read(8192), b''):
534 m.update(chunk)
535 except ValueError:
536 # You can't mmap() an empty file so silence this exception
537 pass
533 return m.hexdigest() 538 return m.hexdigest()
534 539
535def sha256_file(filename): 540def sha256_file(filename):