summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2020-04-08 16:16:53 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-04-24 14:31:42 +0100
commit8b1636763d6df9d68a50ed017ff0ed5f358b7321 (patch)
tree2956f155f84165d73fe74bf760509aab48fc40ff /bitbake
parentab26fdae9e5ae56bb84196698d3fa4fd568fe903 (diff)
downloadpoky-8b1636763d6df9d68a50ed017ff0ed5f358b7321.tar.gz
bitbake: cache: Fix performance problem with large numbers of source files
Some companies are using large numbers of patch files in SRC_URI. Rightly or wrongly that exposes a performance problem where the code does not handle the large string manipulations in a way which works efficienty in python. This is a modified version of a patch from z00539568 <zhangyifan46@huawei.com153340508@qq.com which addresses the performance problem. I modified it to use a more advanced regex, retain the "*" check and cache the regex. [YOCTO #13824] (Bitbake rev: c07f374998903359ed55f263c86466d05aa39b68) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/cache.py22
1 files changed, 8 insertions, 14 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index a5aaf3b999..d1be83617b 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -21,6 +21,7 @@ import logging
21import pickle 21import pickle
22from collections import defaultdict 22from collections import defaultdict
23import bb.utils 23import bb.utils
24import re
24 25
25logger = logging.getLogger("BitBake.Cache") 26logger = logging.getLogger("BitBake.Cache")
26 27
@@ -369,6 +370,7 @@ class Cache(NoCache):
369 self.data_fn = None 370 self.data_fn = None
370 self.cacheclean = True 371 self.cacheclean = True
371 self.data_hash = data_hash 372 self.data_hash = data_hash
373 self.filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
372 374
373 if self.cachedir in [None, '']: 375 if self.cachedir in [None, '']:
374 self.has_cache = False 376 self.has_cache = False
@@ -607,20 +609,12 @@ class Cache(NoCache):
607 if hasattr(info_array[0], 'file_checksums'): 609 if hasattr(info_array[0], 'file_checksums'):
608 for _, fl in info_array[0].file_checksums.items(): 610 for _, fl in info_array[0].file_checksums.items():
609 fl = fl.strip() 611 fl = fl.strip()
610 while fl: 612 if not fl:
611 # A .split() would be simpler but means spaces or colons in filenames would break 613 continue
612 a = fl.find(":True") 614 # Have to be careful about spaces and colons in filenames
613 b = fl.find(":False") 615 flist = self.filelist_regex.split(fl)
614 if ((a < 0) and b) or ((b > 0) and (b < a)): 616 for f in flist:
615 f = fl[:b+6] 617 if not f or "*" in f:
616 fl = fl[b+7:]
617 elif ((b < 0) and a) or ((a > 0) and (a < b)):
618 f = fl[:a+5]
619 fl = fl[a+6:]
620 else:
621 break
622 fl = fl.strip()
623 if "*" in f:
624 continue 618 continue
625 f, exist = f.split(":") 619 f, exist = f.split(":")
626 if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)): 620 if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):