summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2018-07-23 16:22:55 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-07-25 16:48:27 +0100
commitce5cd33a8a0064a21dce0e10cb93c6fd3e85acdd (patch)
treede2a69304fe165967de841cc57101f0e69fcc818
parenta6a865ba933ca7f8a2f93ac4aeaa918ef42d36b1 (diff)
downloadpoky-ce5cd33a8a0064a21dce0e10cb93c6fd3e85acdd.tar.gz
license_image: Fix race
The current code pokes into do_deploy manifests from do_image_complete when the do_image_complete task may or may not depend upon the do_deploy tasks in question. Often it gets lucky, sometimes it results in build failures. To fix this, split the functionality to its own task which can have the correct task dependencies. This means the data in BB_TASKDEPDATA is definitive, the other code can be dropped, as can the IMAGE_EXTRATYPES do_populate_lic dependencies from image.bbclass. This fixes bugs which show up as: NOTE: recipe linux-yocto-4.14.48+gitAUTOINC+d64aec9793_97c8063d2d-r0: task do_deploy: Started ERROR: core-image-minimal-1.0-r0 do_image_complete: Error executing a python function in exec_python_func() autogenerated: The stack trace of python calls that resulted in this exception/failure was: File: 'exec_python_func() autogenerated', lineno: 2, function: <module> 0001: *** 0002:write_deploy_manifest(d) 0003: File: '/home/pokybuild/yocto-autobuilder/yocto-worker/nightly-oe-selftest/build/meta/classes/license.bbclass', lineno: 33, function: write_deploy_manifest 0029: 'w+').write(output) 0030:} 0031: 0032:python write_deploy_manifest() { *** 0033: license_deployed_manifest(d) 0034:} 0035: 0036:python license_create_manifest() { 0037: import oe.packagedata File: '/home/pokybuild/yocto-autobuilder/yocto-worker/nightly-oe-selftest/build/meta/classes/license.bbclass', lineno: 191, function: license_deployed_manifest 0187: # It is necessary to mark this will be used for image manifest 0188: man_dic[dep]["IMAGE_MANIFEST"] = True 0189: man_dic[dep]["PN"] = dep 0190: man_dic[dep]["FILES"] = \ *** 0191: " ".join(get_deployed_files(dep_dic[dep])) 0192: with open(os.path.join(lic_dir, dep, "recipeinfo"), "r") as f: 0193: for line in f.readlines(): 0194: key,val = line.split(": ", 1) 0195: man_dic[dep][key] = val[:-1] File: '/home/pokybuild/yocto-autobuilder/yocto-worker/nightly-oe-selftest/build/meta/classes/license.bbclass', lineno: 289, function: get_deployed_files 0285: """ 0286: 0287: dep_files = [] 0288: excluded_files = [] *** 0289: with open(man_file, "r") as manifest: 0290: all_files = manifest.read() 0291: for f in all_files.splitlines(): 0292: if ((not (os.path.islink(f) or os.path.isdir(f))) and 0293: not os.path.basename(f) in excluded_files): Exception: FileNotFoundError: [Errno 2] No such file or directory: '/home/pokybuild/yocto-autobuilder/yocto-worker/nightly-oe-selftest/build/build-st-730/tmp/sstate-control/manifest-qemux86_64-linux-yocto.deploy' ERROR: core-image-minimal-1.0-r0 do_image_complete: Function failed: write_deploy_manifest ERROR: Logfile of failure stored in: /home/pokybuild/yocto-autobuilder/yocto-worker/nightly-oe-selftest/build/build-st-730/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/temp/log.do_image_complete.50537 NOTE: recipe core-image-minimal-1.0-r0: task do_image_complete: Failed ERROR: Task (/home/pokybuild/yocto-autobuilder/yocto-worker/nightly-oe-selftest/build/meta/recipes-core/images/core-image-minimal.bb:do_image_complete) failed with exit code '1' NOTE: recipe linux-yocto-4.14.48+gitAUTOINC+d64aec9793_97c8063d2d-r0: task do_deploy: Succeeded (From OE-Core rev: b54cdaea7844ee3bf0c39eb97cc7c4c17ed5818c) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/image.bbclass1
-rw-r--r--meta/classes/license_image.bbclass58
2 files changed, 6 insertions, 53 deletions
diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index adc50c99bc..c3e73676dc 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -141,7 +141,6 @@ python () {
141 deps += " %s:%s" % (dep, task) 141 deps += " %s:%s" % (dep, task)
142 return deps 142 return deps
143 143
144 d.appendVarFlag('do_image', 'depends', extraimage_getdepends('do_populate_lic'))
145 d.appendVarFlag('do_image_complete', 'depends', extraimage_getdepends('do_populate_sysroot')) 144 d.appendVarFlag('do_image_complete', 'depends', extraimage_getdepends('do_populate_sysroot'))
146 145
147 deps = " " + imagetypes_getdepends(d) 146 deps = " " + imagetypes_getdepends(d)
diff --git a/meta/classes/license_image.bbclass b/meta/classes/license_image.bbclass
index efeedce519..f0fbb763f1 100644
--- a/meta/classes/license_image.bbclass
+++ b/meta/classes/license_image.bbclass
@@ -11,10 +11,6 @@ python write_package_manifest() {
11 'w+').write(output) 11 'w+').write(output)
12} 12}
13 13
14python write_deploy_manifest() {
15 license_deployed_manifest(d)
16}
17
18python license_create_manifest() { 14python license_create_manifest() {
19 import oe.packagedata 15 import oe.packagedata
20 from oe.rootfs import image_list_installed_packages 16 from oe.rootfs import image_list_installed_packages
@@ -197,11 +193,6 @@ def get_deployed_dependencies(d):
197 depends = list(set([dep[0] for dep 193 depends = list(set([dep[0] for dep
198 in list(taskdata.values()) 194 in list(taskdata.values())
199 if not dep[0].endswith("-native")])) 195 if not dep[0].endswith("-native")]))
200 extra_depends = d.getVar("EXTRA_IMAGEDEPENDS")
201 boot_depends = get_boot_dependencies(d)
202 depends.extend(extra_depends.split())
203 depends.extend(boot_depends)
204 depends = list(set(depends))
205 196
206 # To verify what was deployed it checks the rootfs dependencies against 197 # To verify what was deployed it checks the rootfs dependencies against
207 # the SSTATE_MANIFESTS for "deploy" task. 198 # the SSTATE_MANIFESTS for "deploy" task.
@@ -210,15 +201,6 @@ def get_deployed_dependencies(d):
210 sstate_manifest_dir = d.getVar("SSTATE_MANIFESTS") 201 sstate_manifest_dir = d.getVar("SSTATE_MANIFESTS")
211 archs = list(set(d.getVar("SSTATE_ARCHS").split())) 202 archs = list(set(d.getVar("SSTATE_ARCHS").split()))
212 for dep in depends: 203 for dep in depends:
213 # Some recipes have an arch on their own, so we try that first.
214 special_arch = d.getVar("PACKAGE_ARCH_pn-%s" % dep)
215 if special_arch:
216 sstate_manifest_file = os.path.join(sstate_manifest_dir,
217 "manifest-%s-%s.deploy" % (special_arch, dep))
218 if os.path.exists(sstate_manifest_file):
219 deploy[dep] = sstate_manifest_file
220 continue
221
222 for arch in archs: 204 for arch in archs:
223 sstate_manifest_file = os.path.join(sstate_manifest_dir, 205 sstate_manifest_file = os.path.join(sstate_manifest_dir,
224 "manifest-%s-%s.deploy" % (arch, dep)) 206 "manifest-%s-%s.deploy" % (arch, dep))
@@ -229,38 +211,6 @@ def get_deployed_dependencies(d):
229 return deploy 211 return deploy
230get_deployed_dependencies[vardepsexclude] = "BB_TASKDEPDATA" 212get_deployed_dependencies[vardepsexclude] = "BB_TASKDEPDATA"
231 213
232def get_boot_dependencies(d):
233 """
234 Return the dependencies from boot tasks
235 """
236
237 depends = []
238 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
239 # Only bootimg includes the depends flag
240 boot_depends_string = d.getVarFlag("do_bootimg", "depends") or ""
241 boot_depends = [dep.split(":")[0] for dep
242 in boot_depends_string.split()
243 if not dep.split(":")[0].endswith("-native")]
244 for dep in boot_depends:
245 info_file = os.path.join(d.getVar("LICENSE_DIRECTORY"),
246 dep, "recipeinfo")
247 # If the recipe and dependency name is the same
248 if os.path.exists(info_file):
249 depends.append(dep)
250 # We need to search for the provider of the dependency
251 else:
252 for taskdep in taskdepdata.values():
253 # The fifth field contains what the task provides
254 if dep in taskdep[4]:
255 info_file = os.path.join(
256 d.getVar("LICENSE_DIRECTORY"),
257 taskdep[0], "recipeinfo")
258 if os.path.exists(info_file):
259 depends.append(taskdep[0])
260 break
261 return depends
262get_boot_dependencies[vardepsexclude] = "BB_TASKDEPDATA"
263
264def get_deployed_files(man_file): 214def get_deployed_files(man_file):
265 """ 215 """
266 Get the files deployed from the sstate manifest 216 Get the files deployed from the sstate manifest
@@ -279,6 +229,10 @@ def get_deployed_files(man_file):
279ROOTFS_POSTPROCESS_COMMAND_prepend = "write_package_manifest; license_create_manifest; " 229ROOTFS_POSTPROCESS_COMMAND_prepend = "write_package_manifest; license_create_manifest; "
280do_rootfs[recrdeptask] += "do_populate_lic" 230do_rootfs[recrdeptask] += "do_populate_lic"
281 231
282IMAGE_POSTPROCESS_COMMAND_prepend = "write_deploy_manifest; " 232python do_populate_lic_deploy() {
283do_image[recrdeptask] += "do_populate_lic" 233 license_deployed_manifest(d)
234}
235
236addtask populate_lic_deploy before do_build after do_image_complete
237do_populate_lic_deploy[recrdeptask] += "do_populate_lic do_deploy"
284 238