From 411e2b7047b04edb9065f84d96cc7d14224c5032 Mon Sep 17 00:00:00 2001 From: Alexander Kanavin Date: Wed, 23 Jan 2019 17:17:43 +0100 Subject: lib/oe/reciputils.py: parallelize upstream version checks Previously this was done via bitbake tasks, and when this was rewritten to a for loop, performance sufered significantly: from 90 seconds to about 12 minutes for oe-core. This change restores the previous run time, and makes it possible to perform such checks with command line utilities in an interactive way. Implementation note: we have to create a copy of the recipe data, as Tinfoil API can't be used from multiple threads and only allows one process to access the data at a time. (From OE-Core rev: b1d01911fa2a0a4945da071d66fb50e9f14ded81) Signed-off-by: Alexander Kanavin Signed-off-by: Richard Purdie --- meta/lib/oe/recipeutils.py | 77 +++++++++++++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 21 deletions(-) diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py index 39d3de4bb1..92c0f65257 100644 --- a/meta/lib/oe/recipeutils.py +++ b/meta/lib/oe/recipeutils.py @@ -1020,8 +1020,54 @@ def get_recipe_upstream_version(rd): return ru +def _get_recipe_upgrade_status(data): + uv = get_recipe_upstream_version(data) + + pn = data.getVar('PN') + cur_ver = uv['current_version'] + + upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN') + if not uv['version']: + status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" + else: + cmp = vercmp_string(uv['current_version'], uv['version']) + if cmp == -1: + status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN" + elif cmp == 0: + status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN" + else: + status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" + + next_ver = uv['version'] if uv['version'] else "N/A" + revision = uv['revision'] if uv['revision'] else "N/A" + maintainer = data.getVar('RECIPE_MAINTAINER') + no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON') + + return (pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason) + def get_recipe_upgrade_status(recipes=None): pkgs_list = [] + data_copy_list = [] + copy_vars = ('SRC_URI', + 'PV', + 'GITDIR', + 'DL_DIR', + 'PN', + 'CACHE', + 'PERSISTENT_DIR', + 'BB_URI_HEADREVS', + 'UPSTREAM_CHECK_COMMITS', + 'UPSTREAM_CHECK_GITTAGREGEX', + 'UPSTREAM_CHECK_REGEX', + 'UPSTREAM_CHECK_URI', + 'UPSTREAM_VERSION_UNKNOWN', + 'RECIPE_MAINTAINER', + 'RECIPE_NO_UPDATE_REASON', + 'RECIPE_UPSTREAM_VERSION', + 'RECIPE_UPSTREAM_DATE', + 'CHECK_DATE', + ) + with bb.tinfoil.Tinfoil() as tinfoil: tinfoil.prepare(config_only=False) @@ -1043,28 +1089,17 @@ def get_recipe_upgrade_status(recipes=None): bb.note(" Skip package %s as upstream check unreliable" % pn) continue - uv = get_recipe_upstream_version(data) - - pn = data.getVar('PN') - cur_ver = uv['current_version'] - - upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN') - if not uv['version']: - status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" - else: - cmp = vercmp_string(uv['current_version'], uv['version']) - if cmp == -1: - status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN" - elif cmp == 0: - status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN" - else: - status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN" + data_copy = bb.data.init() + for var in copy_vars: + data_copy.setVar(var, data.getVar(var)) + for k in data: + if k.startswith('SRCREV'): + data_copy.setVar(k, data.getVar(k)) - next_ver = uv['version'] if uv['version'] else "N/A" - revision = uv['revision'] if uv['revision'] else "N/A" - maintainer = data.getVar('RECIPE_MAINTAINER') - no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON') + data_copy_list.append(data_copy) - pkgs_list.append((pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason)) + from concurrent.futures import ProcessPoolExecutor + with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor: + pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list) return pkgs_list -- cgit v1.2.3-54-g00ecf