summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2015-05-27 17:31:10 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-05-29 10:17:16 +0100
commit08b77c8cef3d93098118b9141c19bba382b30d6f (patch)
tree82bc8453b0710d564b56cca31eeb828c9fa591f4
parent553267d8d9a732dd1d382cc25077b49caae29af6 (diff)
downloadpoky-08b77c8cef3d93098118b9141c19bba382b30d6f.tar.gz
bitbake: cooker/utils: Improve parsing profiling
Currently the cooker parsing processes each dump an individual profile which is ok, but means absolute numbers of function calls for a given load can be tricky to determine as parsing of recipes may go to different pool threads on different runs. This change collects up the individual thread parsing results and processes them into one profile output. The profile processing function in utils needed tweaks to allow this to work. (Bitbake rev: d3d2541aacd1ea560da0d8b25a3ea3f0563dee70) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/cooker.py14
-rw-r--r--bitbake/lib/bb/utils.py12
2 files changed, 21 insertions, 5 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index ddf5fedb83..577d808511 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -1798,8 +1798,6 @@ class Parser(multiprocessing.Process):
1798 finally: 1798 finally:
1799 logfile = "profile-parse-%s.log" % multiprocessing.current_process().name 1799 logfile = "profile-parse-%s.log" % multiprocessing.current_process().name
1800 prof.dump_stats(logfile) 1800 prof.dump_stats(logfile)
1801 bb.utils.process_profilelog(logfile)
1802 print("Raw profiling information saved to %s and processed statistics to %s.processed" % (logfile, logfile))
1803 1801
1804 def realrun(self): 1802 def realrun(self):
1805 if self.init: 1803 if self.init:
@@ -1869,6 +1867,7 @@ class CookerParser(object):
1869 self.current = 0 1867 self.current = 0
1870 self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or 1868 self.num_processes = int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS", True) or
1871 multiprocessing.cpu_count()) 1869 multiprocessing.cpu_count())
1870 self.process_names = []
1872 1871
1873 self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array) 1872 self.bb_cache = bb.cache.Cache(self.cfgdata, self.cfghash, cooker.caches_array)
1874 self.fromcache = [] 1873 self.fromcache = []
@@ -1904,6 +1903,7 @@ class CookerParser(object):
1904 for i in range(0, self.num_processes): 1903 for i in range(0, self.num_processes):
1905 parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) 1904 parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
1906 parser.start() 1905 parser.start()
1906 self.process_names.append(parser.name)
1907 self.processes.append(parser) 1907 self.processes.append(parser)
1908 1908
1909 self.results = itertools.chain(self.results, self.parse_generator()) 1909 self.results = itertools.chain(self.results, self.parse_generator())
@@ -1947,6 +1947,16 @@ class CookerParser(object):
1947 multiprocessing.util.Finalize(None, sync.join, exitpriority=-100) 1947 multiprocessing.util.Finalize(None, sync.join, exitpriority=-100)
1948 bb.codeparser.parser_cache_savemerge(self.cooker.data) 1948 bb.codeparser.parser_cache_savemerge(self.cooker.data)
1949 bb.fetch.fetcher_parse_done(self.cooker.data) 1949 bb.fetch.fetcher_parse_done(self.cooker.data)
1950 if self.cooker.configuration.profile:
1951 profiles = []
1952 for i in self.process_names:
1953 logfile = "profile-parse-%s.log" % i
1954 if os.path.exists(logfile):
1955 profiles.append(logfile)
1956
1957 pout = "profile-parse.log.processed"
1958 bb.utils.process_profilelog(profiles, pout = pout)
1959 print("Processed parsing statistics saved to %s" % (pout))
1950 1960
1951 def load_cached(self): 1961 def load_cached(self):
1952 for filename, appends in self.fromcache: 1962 for filename, appends in self.fromcache:
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index 988b845a4a..857f5bcf96 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -908,11 +908,17 @@ def cpu_count():
908def nonblockingfd(fd): 908def nonblockingfd(fd):
909 fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK) 909 fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
910 910
911def process_profilelog(fn): 911def process_profilelog(fn, pout = None):
912 pout = open(fn + '.processed', 'w') 912 # Either call with a list of filenames and set pout or a filename and optionally pout.
913 if not pout:
914 pout = fn + '.processed'
915 pout = open(pout, 'w')
913 916
914 import pstats 917 import pstats
915 p = pstats.Stats(fn, stream=pout) 918 if isinstance(fn, list):
919 p = pstats.Stats(*fn, stream=pout)
920 else:
921 p = pstats.Stats(fn, stream=pout)
916 p.sort_stats('time') 922 p.sort_stats('time')
917 p.print_stats() 923 p.print_stats()
918 p.print_callers() 924 p.print_callers()