diff options
author | Jose Quaresma <quaresma.jose@gmail.com> | 2022-04-16 23:28:45 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2022-07-15 17:51:34 +0100 |
commit | 6421d3330c10868faf01084d4fdd1f3f9874f3bd (patch) | |
tree | 51f716335b644be8979f4b8719347cf7f6f394ae /meta/classes/sstate.bbclass | |
parent | a2d8217488e51496caf1a6bf435c32a8f80faf07 (diff) | |
download | poky-6421d3330c10868faf01084d4fdd1f3f9874f3bd.tar.gz |
sstate: Use the python3 ThreadPoolExecutor instead of the OE ThreadedPool
For the FetchConnectionCache use a queue where each thread can
get an unsed connection_cache that is properly initialized before
we fireup the ThreadPoolExecutor.
(From OE-Core rev: eb6a6820928472ef194b963b606454e731f9486f)
Signed-off-by: Jose Quaresma <quaresma.jose@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/classes/sstate.bbclass')
-rw-r--r-- | meta/classes/sstate.bbclass | 32 |
1 files changed, 19 insertions, 13 deletions
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass index 3513269bca..0aa901fe89 100644 --- a/meta/classes/sstate.bbclass +++ b/meta/classes/sstate.bbclass | |||
@@ -977,15 +977,19 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, | |||
977 | localdata.delVar('BB_NO_NETWORK') | 977 | localdata.delVar('BB_NO_NETWORK') |
978 | 978 | ||
979 | from bb.fetch2 import FetchConnectionCache | 979 | from bb.fetch2 import FetchConnectionCache |
980 | def checkstatus_init(thread_worker): | 980 | def checkstatus_init(): |
981 | thread_worker.connection_cache = FetchConnectionCache() | 981 | while not connection_cache_pool.full(): |
982 | connection_cache_pool.put(FetchConnectionCache()) | ||
982 | 983 | ||
983 | def checkstatus_end(thread_worker): | 984 | def checkstatus_end(): |
984 | thread_worker.connection_cache.close_connections() | 985 | while not connection_cache_pool.empty(): |
986 | connection_cache = connection_cache_pool.get() | ||
987 | connection_cache.close_connections() | ||
985 | 988 | ||
986 | def checkstatus(thread_worker, arg): | 989 | def checkstatus(arg): |
987 | (tid, sstatefile) = arg | 990 | (tid, sstatefile) = arg |
988 | 991 | ||
992 | connection_cache = connection_cache_pool.get() | ||
989 | localdata2 = bb.data.createCopy(localdata) | 993 | localdata2 = bb.data.createCopy(localdata) |
990 | srcuri = "file://" + sstatefile | 994 | srcuri = "file://" + sstatefile |
991 | localdata2.setVar('SRC_URI', srcuri) | 995 | localdata2.setVar('SRC_URI', srcuri) |
@@ -995,7 +999,7 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, | |||
995 | 999 | ||
996 | try: | 1000 | try: |
997 | fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2, | 1001 | fetcher = bb.fetch2.Fetch(srcuri.split(), localdata2, |
998 | connection_cache=thread_worker.connection_cache) | 1002 | connection_cache=connection_cache) |
999 | fetcher.checkstatus() | 1003 | fetcher.checkstatus() |
1000 | bb.debug(2, "SState: Successful fetch test for %s" % srcuri) | 1004 | bb.debug(2, "SState: Successful fetch test for %s" % srcuri) |
1001 | found.add(tid) | 1005 | found.add(tid) |
@@ -1005,6 +1009,8 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, | |||
1005 | except Exception as e: | 1009 | except Exception as e: |
1006 | bb.error("SState: cannot test %s: %s\n%s" % (srcuri, repr(e), traceback.format_exc())) | 1010 | bb.error("SState: cannot test %s: %s\n%s" % (srcuri, repr(e), traceback.format_exc())) |
1007 | 1011 | ||
1012 | connection_cache_pool.put(connection_cache) | ||
1013 | |||
1008 | if progress: | 1014 | if progress: |
1009 | bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d) | 1015 | bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d) |
1010 | 1016 | ||
@@ -1025,13 +1031,13 @@ def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, summary=True, | |||
1025 | fetcherenv = bb.fetch2.get_fetcher_environment(d) | 1031 | fetcherenv = bb.fetch2.get_fetcher_environment(d) |
1026 | with bb.utils.environment(**fetcherenv): | 1032 | with bb.utils.environment(**fetcherenv): |
1027 | bb.event.enable_threadlock() | 1033 | bb.event.enable_threadlock() |
1028 | pool = oe.utils.ThreadedPool(nproc, len(tasklist), | 1034 | import concurrent.futures |
1029 | worker_init=checkstatus_init, worker_end=checkstatus_end, | 1035 | from queue import Queue |
1030 | name="sstate_checkhashes-") | 1036 | connection_cache_pool = Queue(nproc) |
1031 | for t in tasklist: | 1037 | checkstatus_init() |
1032 | pool.add_task(checkstatus, t) | 1038 | with concurrent.futures.ThreadPoolExecutor(max_workers=nproc) as executor: |
1033 | pool.start() | 1039 | executor.map(checkstatus, tasklist.copy()) |
1034 | pool.wait_completion() | 1040 | checkstatus_end() |
1035 | bb.event.disable_threadlock() | 1041 | bb.event.disable_threadlock() |
1036 | 1042 | ||
1037 | if progress: | 1043 | if progress: |