license_image: Fix race

The current code pokes into do_deploy manifests from do_image_complete when the
do_image_complete task may or may not depend upon the do_deploy tasks in question.
Often it gets lucky, sometimes it results in build failures.

To fix this, split the functionality to its own task which can have the correct
task dependencies. This means the data in BB_TASKDEPDATA is definitive, the other
code can be dropped, as can the IMAGE_EXTRATYPES do_populate_lic dependencies
from image.bbclass.

This fixes bugs which show up as:

NOTE: recipe linux-yocto-4.14.48+gitAUTOINC+d64aec9793_97c8063d2d-r0: task do_deploy: Started
ERROR: core-image-minimal-1.0-r0 do_image_complete: Error executing a python function in exec_python_func() autogenerated:

The stack trace of python calls that resulted in this exception/failure was:
File: 'exec_python_func() autogenerated', lineno: 2, function: <module>
     0001:
 *** 0002:write_deploy_manifest(d)
     0003:
File: '/home/pokybuild/yocto-autobuilder/yocto-worker/nightly-oe-selftest/build/meta/classes/license.bbclass', lineno: 33, function: write_deploy_manifest
     0029:        'w+').write(output)
     0030:}
     0031:
     0032:python write_deploy_manifest() {
 *** 0033:    license_deployed_manifest(d)
     0034:}
     0035:
     0036:python license_create_manifest() {
     0037:    import oe.packagedata
File: '/home/pokybuild/yocto-autobuilder/yocto-worker/nightly-oe-selftest/build/meta/classes/license.bbclass', lineno: 191, function: license_deployed_manifest
     0187:        # It is necessary to mark this will be used for image manifest
     0188:        man_dic[dep]["IMAGE_MANIFEST"] = True
     0189:        man_dic[dep]["PN"] = dep
     0190:        man_dic[dep]["FILES"] = \
 *** 0191:            " ".join(get_deployed_files(dep_dic[dep]))
     0192:        with open(os.path.join(lic_dir, dep, "recipeinfo"), "r") as f:
     0193:            for line in f.readlines():
     0194:                key,val = line.split(": ", 1)
     0195:                man_dic[dep][key] = val[:-1]
File: '/home/pokybuild/yocto-autobuilder/yocto-worker/nightly-oe-selftest/build/meta/classes/license.bbclass', lineno: 289, function: get_deployed_files
     0285:    """
     0286:
     0287:    dep_files = []
     0288:    excluded_files = []
 *** 0289:    with open(man_file, "r") as manifest:
     0290:        all_files = manifest.read()
     0291:    for f in all_files.splitlines():
     0292:        if ((not (os.path.islink(f) or os.path.isdir(f))) and
     0293:                not os.path.basename(f) in excluded_files):
Exception: FileNotFoundError: [Errno 2] No such file or directory: '/home/pokybuild/yocto-autobuilder/yocto-worker/nightly-oe-selftest/build/build-st-730/tmp/sstate-control/manifest-qemux86_64-linux-yocto.deploy'

ERROR: core-image-minimal-1.0-r0 do_image_complete: Function failed: write_deploy_manifest
ERROR: Logfile of failure stored in: /home/pokybuild/yocto-autobuilder/yocto-worker/nightly-oe-selftest/build/build-st-730/tmp/work/qemux86_64-poky-linux/core-image-minimal/1.0-r0/temp/log.do_image_complete.50537
NOTE: recipe core-image-minimal-1.0-r0: task do_image_complete: Failed
ERROR: Task (/home/pokybuild/yocto-autobuilder/yocto-worker/nightly-oe-selftest/build/meta/recipes-core/images/core-image-minimal.bb:do_image_complete) failed with exit code '1'
NOTE: recipe linux-yocto-4.14.48+gitAUTOINC+d64aec9793_97c8063d2d-r0: task do_deploy: Succeeded

(From OE-Core rev: b54cdaea7844ee3bf0c39eb97cc7c4c17ed5818c)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

2 files changed, 6 insertions, 53 deletions

diff --git a/meta/classes/image.bbclass b/meta/classes/image.bbclass
index adc50c99bc..c3e73676dc 100644
--- a/meta/classes/image.bbclass
+++ b/meta/classes/image.bbclass
@@ -141,7 +141,6 @@ python () {
             deps += " %s:%s" % (dep, task)
         return deps
 
-    d.appendVarFlag('do_image', 'depends', extraimage_getdepends('do_populate_lic'))
     d.appendVarFlag('do_image_complete', 'depends', extraimage_getdepends('do_populate_sysroot'))
 
     deps = " " + imagetypes_getdepends(d)
diff --git a/meta/classes/license_image.bbclass b/meta/classes/license_image.bbclass
index efeedce519..f0fbb763f1 100644
--- a/meta/classes/license_image.bbclass
+++ b/meta/classes/license_image.bbclass
@@ -11,10 +11,6 @@ python write_package_manifest() {
         'w+').write(output)
 }
 
-python write_deploy_manifest() {
-    license_deployed_manifest(d)
-}
-
 python license_create_manifest() {
     import oe.packagedata
     from oe.rootfs import image_list_installed_packages
@@ -197,11 +193,6 @@ def get_deployed_dependencies(d):
     depends = list(set([dep[0] for dep
                     in list(taskdata.values())
                     if not dep[0].endswith("-native")]))
-    extra_depends = d.getVar("EXTRA_IMAGEDEPENDS")
-    boot_depends = get_boot_dependencies(d)
-    depends.extend(extra_depends.split())
-    depends.extend(boot_depends)
-    depends = list(set(depends))
 
     # To verify what was deployed it checks the rootfs dependencies against
     # the SSTATE_MANIFESTS for "deploy" task.
@@ -210,15 +201,6 @@ def get_deployed_dependencies(d):
     sstate_manifest_dir = d.getVar("SSTATE_MANIFESTS")
     archs = list(set(d.getVar("SSTATE_ARCHS").split()))
     for dep in depends:
-        # Some recipes have an arch on their own, so we try that first.
-        special_arch = d.getVar("PACKAGE_ARCH_pn-%s" % dep)
-        if special_arch:
-            sstate_manifest_file = os.path.join(sstate_manifest_dir,
-                    "manifest-%s-%s.deploy" % (special_arch, dep))
-            if os.path.exists(sstate_manifest_file):
-                deploy[dep] = sstate_manifest_file
-                continue
-
         for arch in archs:
             sstate_manifest_file = os.path.join(sstate_manifest_dir,
                     "manifest-%s-%s.deploy" % (arch, dep))
@@ -229,38 +211,6 @@ def get_deployed_dependencies(d):
     return deploy
 get_deployed_dependencies[vardepsexclude] = "BB_TASKDEPDATA"
 
-def get_boot_dependencies(d):
-    """
-    Return the dependencies from boot tasks
-    """
-
-    depends = []
-    taskdepdata = d.getVar("BB_TASKDEPDATA", False)
-    # Only bootimg includes the depends flag
-    boot_depends_string = d.getVarFlag("do_bootimg", "depends") or ""
-    boot_depends = [dep.split(":")[0] for dep
-                in boot_depends_string.split()
-                if not dep.split(":")[0].endswith("-native")]
-    for dep in boot_depends:
-        info_file = os.path.join(d.getVar("LICENSE_DIRECTORY"),
-                dep, "recipeinfo")
-        # If the recipe and dependency name is the same
-        if os.path.exists(info_file):
-            depends.append(dep)
-        # We need to search for the provider of the dependency
-        else:
-            for taskdep in taskdepdata.values():
-                # The fifth field contains what the task provides
-                if dep in taskdep[4]:
-                    info_file = os.path.join(
-                            d.getVar("LICENSE_DIRECTORY"),
-                            taskdep[0], "recipeinfo")
-                    if os.path.exists(info_file):
-                        depends.append(taskdep[0])
-                        break
-    return depends
-get_boot_dependencies[vardepsexclude] = "BB_TASKDEPDATA"
-
 def get_deployed_files(man_file):
     """
     Get the files deployed from the sstate manifest
@@ -279,6 +229,10 @@ def get_deployed_files(man_file):
 ROOTFS_POSTPROCESS_COMMAND_prepend = "write_package_manifest; license_create_manifest; "
 do_rootfs[recrdeptask] += "do_populate_lic"
 
-IMAGE_POSTPROCESS_COMMAND_prepend = "write_deploy_manifest; "
-do_image[recrdeptask] += "do_populate_lic"
+python do_populate_lic_deploy() {
+    license_deployed_manifest(d)
+}
+
+addtask populate_lic_deploy before do_build after do_image_complete
+do_populate_lic_deploy[recrdeptask] += "do_populate_lic do_deploy"