summaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
authorLeonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>2017-09-04 14:35:48 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-09-11 17:30:28 +0100
commitf6f6b3e5d5a77010d5716b2cb980b271ee5dc8e8 (patch)
treef4d9e7737b858892c911ffe69fa23d8be5aba689 /meta
parenta8ab6d5c829d02107a968e5925c96f78bc98522a (diff)
downloadpoky-f6f6b3e5d5a77010d5716b2cb980b271ee5dc8e8.tar.gz
package_[deb|ipk]: improve multiprocess logic when creating deb/ipk packages
Current implementation does not handle possible exceptions coming from child processes, the latter responsible for creating packages. With the aim to have more control, use pipes to communicate exceptions and stop package creation in case of failure. Helps to debug [YOCTO #12012]. (From OE-Core rev: 11350a67ba137f560d04aa643ff500a7ff112c73) Signed-off-by: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
-rw-r--r--meta/classes/package_deb.bbclass36
-rw-r--r--meta/classes/package_ipk.bbclass36
2 files changed, 66 insertions, 6 deletions
diff --git a/meta/classes/package_deb.bbclass b/meta/classes/package_deb.bbclass
index 30605344f4..5d297939b6 100644
--- a/meta/classes/package_deb.bbclass
+++ b/meta/classes/package_deb.bbclass
@@ -41,7 +41,29 @@ def debian_arch_map(arch, tune):
41 return arch 41 return arch
42 42
43python do_package_deb () { 43python do_package_deb () {
44 from multiprocessing import Process 44
45 import multiprocessing
46 import traceback
47
48 class DebianWritePkgProcess(multiprocessing.Process):
49 def __init__(self, *args, **kwargs):
50 multiprocessing.Process.__init__(self, *args, **kwargs)
51 self._pconn, self._cconn = multiprocessing.Pipe()
52 self._exception = None
53
54 def run(self):
55 try:
56 multiprocessing.Process.run(self)
57 self._cconn.send(None)
58 except Exception as e:
59 tb = traceback.format_exc()
60 self._cconn.send((e, tb))
61
62 @property
63 def exception(self):
64 if self._pconn.poll():
65 self._exception = self._pconn.recv()
66 return self._exception
45 67
46 oldcwd = os.getcwd() 68 oldcwd = os.getcwd()
47 69
@@ -56,20 +78,28 @@ python do_package_deb () {
56 78
57 max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1) 79 max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
58 launched = [] 80 launched = []
81 error = None
59 pkgs = packages.split() 82 pkgs = packages.split()
60 while pkgs: 83 while not error and pkgs:
61 if len(launched) < max_process: 84 if len(launched) < max_process:
62 p = Process(target=deb_write_pkg, args=(pkgs.pop(), d)) 85 p = DebianWritePkgProcess(target=deb_write_pkg, args=(pkgs.pop(), d))
63 p.start() 86 p.start()
64 launched.append(p) 87 launched.append(p)
65 for q in launched: 88 for q in launched:
66 # The finished processes are joined when calling is_alive() 89 # The finished processes are joined when calling is_alive()
67 if not q.is_alive(): 90 if not q.is_alive():
68 launched.remove(q) 91 launched.remove(q)
92 if q.exception:
93 error, traceback = q.exception
94 break
95
69 for p in launched: 96 for p in launched:
70 p.join() 97 p.join()
71 98
72 os.chdir(oldcwd) 99 os.chdir(oldcwd)
100
101 if error:
102 raise error
73} 103}
74do_package_deb[vardeps] += "deb_write_pkg" 104do_package_deb[vardeps] += "deb_write_pkg"
75do_package_deb[vardepsexclude] = "BB_NUMBER_THREADS" 105do_package_deb[vardepsexclude] = "BB_NUMBER_THREADS"
diff --git a/meta/classes/package_ipk.bbclass b/meta/classes/package_ipk.bbclass
index ec90996184..8439cda6dd 100644
--- a/meta/classes/package_ipk.bbclass
+++ b/meta/classes/package_ipk.bbclass
@@ -17,7 +17,29 @@ OPKG_ARGS += "${@['', '--add-exclude ' + ' --add-exclude '.join((d.getVar('PACKA
17OPKGLIBDIR = "${localstatedir}/lib" 17OPKGLIBDIR = "${localstatedir}/lib"
18 18
19python do_package_ipk () { 19python do_package_ipk () {
20 from multiprocessing import Process 20 import multiprocessing
21 import traceback
22
23 class IPKWritePkgProcess(multiprocessing.Process):
24 def __init__(self, *args, **kwargs):
25 multiprocessing.Process.__init__(self, *args, **kwargs)
26 self._pconn, self._cconn = multiprocessing.Pipe()
27 self._exception = None
28
29 def run(self):
30 try:
31 multiprocessing.Process.run(self)
32 self._cconn.send(None)
33 except Exception as e:
34 tb = traceback.format_exc()
35 self._cconn.send((e, tb))
36
37 @property
38 def exception(self):
39 if self._pconn.poll():
40 self._exception = self._pconn.recv()
41 return self._exception
42
21 43
22 oldcwd = os.getcwd() 44 oldcwd = os.getcwd()
23 45
@@ -41,20 +63,28 @@ python do_package_ipk () {
41 63
42 max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1) 64 max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
43 launched = [] 65 launched = []
66 error = None
44 pkgs = packages.split() 67 pkgs = packages.split()
45 while pkgs: 68 while not error and pkgs:
46 if len(launched) < max_process: 69 if len(launched) < max_process:
47 p = Process(target=ipk_write_pkg, args=(pkgs.pop(), d)) 70 p = IPKWritePkgProcess(target=ipk_write_pkg, args=(pkgs.pop(), d))
48 p.start() 71 p.start()
49 launched.append(p) 72 launched.append(p)
50 for q in launched: 73 for q in launched:
51 # The finished processes are joined when calling is_alive() 74 # The finished processes are joined when calling is_alive()
52 if not q.is_alive(): 75 if not q.is_alive():
53 launched.remove(q) 76 launched.remove(q)
77 if q.exception:
78 error, traceback = q.exception
79 break
80
54 for p in launched: 81 for p in launched:
55 p.join() 82 p.join()
56 83
57 os.chdir(oldcwd) 84 os.chdir(oldcwd)
85
86 if error:
87 raise error
58} 88}
59do_package_ipk[vardeps] += "ipk_write_pkg" 89do_package_ipk[vardeps] += "ipk_write_pkg"
60do_package_ipk[vardepsexclude] = "BB_NUMBER_THREADS" 90do_package_ipk[vardepsexclude] = "BB_NUMBER_THREADS"