diff options
Diffstat (limited to 'meta/classes/create-spdx-2.2.bbclass')
-rw-r--r-- | meta/classes/create-spdx-2.2.bbclass | 380 |
1 files changed, 87 insertions, 293 deletions
diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass index 7c8a0b8b0f..795ba1a882 100644 --- a/meta/classes/create-spdx-2.2.bbclass +++ b/meta/classes/create-spdx-2.2.bbclass | |||
@@ -4,36 +4,9 @@ | |||
4 | # SPDX-License-Identifier: GPL-2.0-only | 4 | # SPDX-License-Identifier: GPL-2.0-only |
5 | # | 5 | # |
6 | 6 | ||
7 | DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx" | 7 | inherit spdx-common |
8 | 8 | ||
9 | # The product name that the CVE database uses. Defaults to BPN, but may need to | 9 | SPDX_VERSION = "2.2" |
10 | # be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). | ||
11 | CVE_PRODUCT ??= "${BPN}" | ||
12 | CVE_VERSION ??= "${PV}" | ||
13 | |||
14 | SPDXDIR ??= "${WORKDIR}/spdx" | ||
15 | SPDXDEPLOY = "${SPDXDIR}/deploy" | ||
16 | SPDXWORK = "${SPDXDIR}/work" | ||
17 | SPDXIMAGEWORK = "${SPDXDIR}/image-work" | ||
18 | SPDXSDKWORK = "${SPDXDIR}/sdk-work" | ||
19 | SPDXDEPS = "${SPDXDIR}/deps.json" | ||
20 | |||
21 | SPDX_TOOL_NAME ??= "oe-spdx-creator" | ||
22 | SPDX_TOOL_VERSION ??= "1.0" | ||
23 | |||
24 | SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" | ||
25 | |||
26 | SPDX_INCLUDE_SOURCES ??= "0" | ||
27 | SPDX_ARCHIVE_SOURCES ??= "0" | ||
28 | SPDX_ARCHIVE_PACKAGED ??= "0" | ||
29 | |||
30 | SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" | ||
31 | SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdocs" | ||
32 | SPDX_PRETTY ??= "0" | ||
33 | |||
34 | SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" | ||
35 | |||
36 | SPDX_CUSTOM_ANNOTATION_VARS ??= "" | ||
37 | 10 | ||
38 | SPDX_ORG ??= "OpenEmbedded ()" | 11 | SPDX_ORG ??= "OpenEmbedded ()" |
39 | SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" | 12 | SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" |
@@ -42,27 +15,12 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f | |||
42 | is the contact information for the person or organization who is doing the \ | 15 | is the contact information for the person or organization who is doing the \ |
43 | build." | 16 | build." |
44 | 17 | ||
45 | def extract_licenses(filename): | ||
46 | import re | ||
47 | |||
48 | lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE) | ||
49 | |||
50 | try: | ||
51 | with open(filename, 'rb') as f: | ||
52 | size = min(15000, os.stat(filename).st_size) | ||
53 | txt = f.read(size) | ||
54 | licenses = re.findall(lic_regex, txt) | ||
55 | if licenses: | ||
56 | ascii_licenses = [lic.decode('ascii') for lic in licenses] | ||
57 | return ascii_licenses | ||
58 | except Exception as e: | ||
59 | bb.warn(f"Exception reading {filename}: {e}") | ||
60 | return None | ||
61 | 18 | ||
62 | def get_doc_namespace(d, doc): | 19 | def get_namespace(d, name): |
63 | import uuid | 20 | import uuid |
64 | namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) | 21 | namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) |
65 | return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) | 22 | return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), name, str(uuid.uuid5(namespace_uuid, name))) |
23 | |||
66 | 24 | ||
67 | def create_annotation(d, comment): | 25 | def create_annotation(d, comment): |
68 | from datetime import datetime, timezone | 26 | from datetime import datetime, timezone |
@@ -80,31 +38,16 @@ def recipe_spdx_is_native(d, recipe): | |||
80 | a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and | 38 | a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and |
81 | a.comment == "isNative" for a in recipe.annotations) | 39 | a.comment == "isNative" for a in recipe.annotations) |
82 | 40 | ||
83 | def is_work_shared_spdx(d): | ||
84 | return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR')) | ||
85 | |||
86 | def get_json_indent(d): | 41 | def get_json_indent(d): |
87 | if d.getVar("SPDX_PRETTY") == "1": | 42 | if d.getVar("SPDX_PRETTY") == "1": |
88 | return 2 | 43 | return 2 |
89 | return None | 44 | return None |
90 | 45 | ||
91 | python() { | ||
92 | import json | ||
93 | if d.getVar("SPDX_LICENSE_DATA"): | ||
94 | return | ||
95 | |||
96 | with open(d.getVar("SPDX_LICENSES"), "r") as f: | ||
97 | data = json.load(f) | ||
98 | # Transform the license array to a dictionary | ||
99 | data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} | ||
100 | d.setVar("SPDX_LICENSE_DATA", data) | ||
101 | } | ||
102 | 46 | ||
103 | def convert_license_to_spdx(lic, document, d, existing={}): | 47 | def convert_license_to_spdx(lic, license_data, document, d, existing={}): |
104 | from pathlib import Path | 48 | from pathlib import Path |
105 | import oe.spdx | 49 | import oe.spdx |
106 | 50 | ||
107 | license_data = d.getVar("SPDX_LICENSE_DATA") | ||
108 | extracted = {} | 51 | extracted = {} |
109 | 52 | ||
110 | def add_extracted_license(ident, name): | 53 | def add_extracted_license(ident, name): |
@@ -172,37 +115,10 @@ def convert_license_to_spdx(lic, document, d, existing={}): | |||
172 | 115 | ||
173 | return ' '.join(convert(l) for l in lic_split) | 116 | return ' '.join(convert(l) for l in lic_split) |
174 | 117 | ||
175 | def process_sources(d): | ||
176 | pn = d.getVar('PN') | ||
177 | assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() | ||
178 | if pn in assume_provided: | ||
179 | for p in d.getVar("PROVIDES").split(): | ||
180 | if p != pn: | ||
181 | pn = p | ||
182 | break | ||
183 | |||
184 | # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted, | ||
185 | # so avoid archiving source here. | ||
186 | if pn.startswith('glibc-locale'): | ||
187 | return False | ||
188 | if d.getVar('PN') == "libtool-cross": | ||
189 | return False | ||
190 | if d.getVar('PN') == "libgcc-initial": | ||
191 | return False | ||
192 | if d.getVar('PN') == "shadow-sysroot": | ||
193 | return False | ||
194 | |||
195 | # We just archive gcc-source for all the gcc related recipes | ||
196 | if d.getVar('BPN') in ['gcc', 'libgcc']: | ||
197 | bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) | ||
198 | return False | ||
199 | |||
200 | return True | ||
201 | |||
202 | |||
203 | def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): | 118 | def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): |
204 | from pathlib import Path | 119 | from pathlib import Path |
205 | import oe.spdx | 120 | import oe.spdx |
121 | import oe.spdx_common | ||
206 | import hashlib | 122 | import hashlib |
207 | 123 | ||
208 | source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") | 124 | source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") |
@@ -255,7 +171,7 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv | |||
255 | )) | 171 | )) |
256 | 172 | ||
257 | if "SOURCE" in spdx_file.fileTypes: | 173 | if "SOURCE" in spdx_file.fileTypes: |
258 | extracted_lics = extract_licenses(filepath) | 174 | extracted_lics = oe.spdx_common.extract_licenses(filepath) |
259 | if extracted_lics: | 175 | if extracted_lics: |
260 | spdx_file.licenseInfoInFiles = extracted_lics | 176 | spdx_file.licenseInfoInFiles = extracted_lics |
261 | 177 | ||
@@ -313,7 +229,8 @@ def add_package_sources_from_debug(d, package_doc, spdx_package, package, packag | |||
313 | debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') | 229 | debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') |
314 | else: | 230 | else: |
315 | debugsrc_path = search / debugsrc.lstrip("/") | 231 | debugsrc_path = search / debugsrc.lstrip("/") |
316 | if not debugsrc_path.exists(): | 232 | # We can only hash files below, skip directories, links, etc. |
233 | if not os.path.isfile(debugsrc_path): | ||
317 | continue | 234 | continue |
318 | 235 | ||
319 | file_sha256 = bb.utils.sha256_file(debugsrc_path) | 236 | file_sha256 = bb.utils.sha256_file(debugsrc_path) |
@@ -346,16 +263,15 @@ def collect_dep_recipes(d, doc, spdx_recipe): | |||
346 | from pathlib import Path | 263 | from pathlib import Path |
347 | import oe.sbom | 264 | import oe.sbom |
348 | import oe.spdx | 265 | import oe.spdx |
266 | import oe.spdx_common | ||
349 | 267 | ||
350 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) | 268 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) |
351 | spdx_deps_file = Path(d.getVar("SPDXDEPS")) | 269 | package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() |
352 | package_archs = d.getVar("SSTATE_ARCHS").split() | ||
353 | package_archs.reverse() | 270 | package_archs.reverse() |
354 | 271 | ||
355 | dep_recipes = [] | 272 | dep_recipes = [] |
356 | 273 | ||
357 | with spdx_deps_file.open("r") as f: | 274 | deps = oe.spdx_common.get_spdx_deps(d) |
358 | deps = json.load(f) | ||
359 | 275 | ||
360 | for dep_pn, dep_hashfn, in_taskhash in deps: | 276 | for dep_pn, dep_hashfn, in_taskhash in deps: |
361 | # If this dependency is not calculated in the taskhash skip it. | 277 | # If this dependency is not calculated in the taskhash skip it. |
@@ -395,7 +311,7 @@ def collect_dep_recipes(d, doc, spdx_recipe): | |||
395 | 311 | ||
396 | return dep_recipes | 312 | return dep_recipes |
397 | 313 | ||
398 | collect_dep_recipes[vardepsexclude] = "SSTATE_ARCHS" | 314 | collect_dep_recipes[vardepsexclude] = "SPDX_MULTILIB_SSTATE_ARCHS" |
399 | 315 | ||
400 | def collect_dep_sources(d, dep_recipes): | 316 | def collect_dep_sources(d, dep_recipes): |
401 | import oe.sbom | 317 | import oe.sbom |
@@ -468,61 +384,24 @@ def add_download_packages(d, doc, recipe): | |||
468 | # but this should be sufficient for now | 384 | # but this should be sufficient for now |
469 | doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) | 385 | doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe) |
470 | 386 | ||
471 | def collect_direct_deps(d, dep_task): | 387 | def get_license_list_version(license_data, d): |
472 | current_task = "do_" + d.getVar("BB_CURRENTTASK") | 388 | # Newer versions of the SPDX license list are SemVer ("MAJOR.MINOR.MICRO"), |
473 | pn = d.getVar("PN") | 389 | # but SPDX 2 only uses "MAJOR.MINOR". |
474 | 390 | return ".".join(license_data["licenseListVersion"].split(".")[:2]) | |
475 | taskdepdata = d.getVar("BB_TASKDEPDATA", False) | ||
476 | |||
477 | for this_dep in taskdepdata.values(): | ||
478 | if this_dep[0] == pn and this_dep[1] == current_task: | ||
479 | break | ||
480 | else: | ||
481 | bb.fatal(f"Unable to find this {pn}:{current_task} in taskdepdata") | ||
482 | |||
483 | deps = set() | ||
484 | for dep_name in this_dep[3]: | ||
485 | dep_data = taskdepdata[dep_name] | ||
486 | if dep_data[1] == dep_task and dep_data[0] != pn: | ||
487 | deps.add((dep_data[0], dep_data[7], dep_name in this_dep[8])) | ||
488 | |||
489 | return sorted(deps) | ||
490 | 391 | ||
491 | collect_direct_deps[vardepsexclude] += "BB_TASKDEPDATA" | ||
492 | collect_direct_deps[vardeps] += "DEPENDS" | ||
493 | |||
494 | python do_collect_spdx_deps() { | ||
495 | # This task calculates the build time dependencies of the recipe, and is | ||
496 | # required because while a task can deptask on itself, those dependencies | ||
497 | # do not show up in BB_TASKDEPDATA. To work around that, this task does the | ||
498 | # deptask on do_create_spdx and writes out the dependencies it finds, then | ||
499 | # do_create_spdx reads in the found dependencies when writing the actual | ||
500 | # SPDX document | ||
501 | import json | ||
502 | from pathlib import Path | ||
503 | |||
504 | spdx_deps_file = Path(d.getVar("SPDXDEPS")) | ||
505 | |||
506 | deps = collect_direct_deps(d, "do_create_spdx") | ||
507 | |||
508 | with spdx_deps_file.open("w") as f: | ||
509 | json.dump(deps, f) | ||
510 | } | ||
511 | # NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source | ||
512 | addtask do_collect_spdx_deps after do_unpack | ||
513 | do_collect_spdx_deps[depends] += "${PATCHDEPENDENCY}" | ||
514 | do_collect_spdx_deps[deptask] = "do_create_spdx" | ||
515 | do_collect_spdx_deps[dirs] = "${SPDXDIR}" | ||
516 | 392 | ||
517 | python do_create_spdx() { | 393 | python do_create_spdx() { |
518 | from datetime import datetime, timezone | 394 | from datetime import datetime, timezone |
519 | import oe.sbom | 395 | import oe.sbom |
520 | import oe.spdx | 396 | import oe.spdx |
397 | import oe.spdx_common | ||
521 | import uuid | 398 | import uuid |
522 | from pathlib import Path | 399 | from pathlib import Path |
523 | from contextlib import contextmanager | 400 | from contextlib import contextmanager |
524 | import oe.cve_check | 401 | import oe.cve_check |
525 | 402 | ||
403 | license_data = oe.spdx_common.load_spdx_license_data(d) | ||
404 | |||
526 | @contextmanager | 405 | @contextmanager |
527 | def optional_tarfile(name, guard, mode="w"): | 406 | def optional_tarfile(name, guard, mode="w"): |
528 | import tarfile | 407 | import tarfile |
@@ -551,10 +430,10 @@ python do_create_spdx() { | |||
551 | doc = oe.spdx.SPDXDocument() | 430 | doc = oe.spdx.SPDXDocument() |
552 | 431 | ||
553 | doc.name = "recipe-" + d.getVar("PN") | 432 | doc.name = "recipe-" + d.getVar("PN") |
554 | doc.documentNamespace = get_doc_namespace(d, doc) | 433 | doc.documentNamespace = get_namespace(d, doc.name) |
555 | doc.creationInfo.created = creation_time | 434 | doc.creationInfo.created = creation_time |
556 | doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." | 435 | doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." |
557 | doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] | 436 | doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) |
558 | doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") | 437 | doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") |
559 | doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) | 438 | doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) |
560 | doc.creationInfo.creators.append("Person: N/A ()") | 439 | doc.creationInfo.creators.append("Person: N/A ()") |
@@ -573,7 +452,7 @@ python do_create_spdx() { | |||
573 | 452 | ||
574 | license = d.getVar("LICENSE") | 453 | license = d.getVar("LICENSE") |
575 | if license: | 454 | if license: |
576 | recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) | 455 | recipe.licenseDeclared = convert_license_to_spdx(license, license_data, doc, d) |
577 | 456 | ||
578 | summary = d.getVar("SUMMARY") | 457 | summary = d.getVar("SUMMARY") |
579 | if summary: | 458 | if summary: |
@@ -610,10 +489,10 @@ python do_create_spdx() { | |||
610 | 489 | ||
611 | add_download_packages(d, doc, recipe) | 490 | add_download_packages(d, doc, recipe) |
612 | 491 | ||
613 | if process_sources(d) and include_sources: | 492 | if oe.spdx_common.process_sources(d) and include_sources: |
614 | recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") | 493 | recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") |
615 | with optional_tarfile(recipe_archive, archive_sources) as archive: | 494 | with optional_tarfile(recipe_archive, archive_sources) as archive: |
616 | spdx_get_src(d) | 495 | oe.spdx_common.get_patched_src(d) |
617 | 496 | ||
618 | add_package_files( | 497 | add_package_files( |
619 | d, | 498 | d, |
@@ -655,10 +534,10 @@ python do_create_spdx() { | |||
655 | package_doc = oe.spdx.SPDXDocument() | 534 | package_doc = oe.spdx.SPDXDocument() |
656 | pkg_name = d.getVar("PKG:%s" % package) or package | 535 | pkg_name = d.getVar("PKG:%s" % package) or package |
657 | package_doc.name = pkg_name | 536 | package_doc.name = pkg_name |
658 | package_doc.documentNamespace = get_doc_namespace(d, package_doc) | 537 | package_doc.documentNamespace = get_namespace(d, package_doc.name) |
659 | package_doc.creationInfo.created = creation_time | 538 | package_doc.creationInfo.created = creation_time |
660 | package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." | 539 | package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." |
661 | package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] | 540 | package_doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) |
662 | package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") | 541 | package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") |
663 | package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) | 542 | package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) |
664 | package_doc.creationInfo.creators.append("Person: N/A ()") | 543 | package_doc.creationInfo.creators.append("Person: N/A ()") |
@@ -671,7 +550,7 @@ python do_create_spdx() { | |||
671 | spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) | 550 | spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) |
672 | spdx_package.name = pkg_name | 551 | spdx_package.name = pkg_name |
673 | spdx_package.versionInfo = d.getVar("PV") | 552 | spdx_package.versionInfo = d.getVar("PV") |
674 | spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) | 553 | spdx_package.licenseDeclared = convert_license_to_spdx(package_license, license_data, package_doc, d, found_licenses) |
675 | spdx_package.supplier = d.getVar("SPDX_SUPPLIER") | 554 | spdx_package.supplier = d.getVar("SPDX_SUPPLIER") |
676 | 555 | ||
677 | package_doc.packages.append(spdx_package) | 556 | package_doc.packages.append(spdx_package) |
@@ -716,48 +595,11 @@ do_create_spdx[dirs] = "${SPDXWORK}" | |||
716 | do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" | 595 | do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" |
717 | do_create_spdx[depends] += "${PATCHDEPENDENCY}" | 596 | do_create_spdx[depends] += "${PATCHDEPENDENCY}" |
718 | 597 | ||
719 | def collect_package_providers(d): | ||
720 | from pathlib import Path | ||
721 | import oe.sbom | ||
722 | import oe.spdx | ||
723 | import json | ||
724 | |||
725 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) | ||
726 | |||
727 | providers = {} | ||
728 | |||
729 | deps = collect_direct_deps(d, "do_create_spdx") | ||
730 | deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME"), True)) | ||
731 | |||
732 | for dep_pn, dep_hashfn, _ in deps: | ||
733 | localdata = d | ||
734 | recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata) | ||
735 | if not recipe_data: | ||
736 | localdata = bb.data.createCopy(d) | ||
737 | localdata.setVar("PKGDATA_DIR", "${PKGDATA_DIR_SDK}") | ||
738 | recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata) | ||
739 | |||
740 | for pkg in recipe_data.get("PACKAGES", "").split(): | ||
741 | |||
742 | pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, localdata) | ||
743 | rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items()) | ||
744 | rprovides.add(pkg) | ||
745 | |||
746 | if "PKG" in pkg_data: | ||
747 | pkg = pkg_data["PKG"] | ||
748 | rprovides.add(pkg) | ||
749 | |||
750 | for r in rprovides: | ||
751 | providers[r] = (pkg, dep_hashfn) | ||
752 | |||
753 | return providers | ||
754 | |||
755 | collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" | ||
756 | |||
757 | python do_create_runtime_spdx() { | 598 | python do_create_runtime_spdx() { |
758 | from datetime import datetime, timezone | 599 | from datetime import datetime, timezone |
759 | import oe.sbom | 600 | import oe.sbom |
760 | import oe.spdx | 601 | import oe.spdx |
602 | import oe.spdx_common | ||
761 | import oe.packagedata | 603 | import oe.packagedata |
762 | from pathlib import Path | 604 | from pathlib import Path |
763 | 605 | ||
@@ -767,9 +609,11 @@ python do_create_runtime_spdx() { | |||
767 | 609 | ||
768 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | 610 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") |
769 | 611 | ||
770 | providers = collect_package_providers(d) | 612 | license_data = oe.spdx_common.load_spdx_license_data(d) |
613 | |||
614 | providers = oe.spdx_common.collect_package_providers(d) | ||
771 | pkg_arch = d.getVar("SSTATE_PKGARCH") | 615 | pkg_arch = d.getVar("SSTATE_PKGARCH") |
772 | package_archs = d.getVar("SSTATE_ARCHS").split() | 616 | package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() |
773 | package_archs.reverse() | 617 | package_archs.reverse() |
774 | 618 | ||
775 | if not is_native: | 619 | if not is_native: |
@@ -800,10 +644,10 @@ python do_create_runtime_spdx() { | |||
800 | 644 | ||
801 | runtime_doc = oe.spdx.SPDXDocument() | 645 | runtime_doc = oe.spdx.SPDXDocument() |
802 | runtime_doc.name = "runtime-" + pkg_name | 646 | runtime_doc.name = "runtime-" + pkg_name |
803 | runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) | 647 | runtime_doc.documentNamespace = get_namespace(localdata, runtime_doc.name) |
804 | runtime_doc.creationInfo.created = creation_time | 648 | runtime_doc.creationInfo.created = creation_time |
805 | runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." | 649 | runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." |
806 | runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] | 650 | runtime_doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) |
807 | runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") | 651 | runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") |
808 | runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) | 652 | runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) |
809 | runtime_doc.creationInfo.creators.append("Person: N/A ()") | 653 | runtime_doc.creationInfo.creators.append("Person: N/A ()") |
@@ -875,7 +719,7 @@ python do_create_runtime_spdx() { | |||
875 | oe.sbom.write_doc(d, runtime_doc, pkg_arch, "runtime", spdx_deploy, indent=get_json_indent(d)) | 719 | oe.sbom.write_doc(d, runtime_doc, pkg_arch, "runtime", spdx_deploy, indent=get_json_indent(d)) |
876 | } | 720 | } |
877 | 721 | ||
878 | do_create_runtime_spdx[vardepsexclude] += "OVERRIDES SSTATE_ARCHS" | 722 | do_create_runtime_spdx[vardepsexclude] += "OVERRIDES SPDX_MULTILIB_SSTATE_ARCHS" |
879 | 723 | ||
880 | addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work | 724 | addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work |
881 | SSTATETASKS += "do_create_runtime_spdx" | 725 | SSTATETASKS += "do_create_runtime_spdx" |
@@ -891,60 +735,6 @@ do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" | |||
891 | do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" | 735 | do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" |
892 | do_create_runtime_spdx[rdeptask] = "do_create_spdx" | 736 | do_create_runtime_spdx[rdeptask] = "do_create_spdx" |
893 | 737 | ||
894 | def spdx_get_src(d): | ||
895 | """ | ||
896 | save patched source of the recipe in SPDX_WORKDIR. | ||
897 | """ | ||
898 | import shutil | ||
899 | spdx_workdir = d.getVar('SPDXWORK') | ||
900 | spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') | ||
901 | pn = d.getVar('PN') | ||
902 | |||
903 | workdir = d.getVar("WORKDIR") | ||
904 | |||
905 | try: | ||
906 | # The kernel class functions require it to be on work-shared, so we dont change WORKDIR | ||
907 | if not is_work_shared_spdx(d): | ||
908 | # Change the WORKDIR to make do_unpack do_patch run in another dir. | ||
909 | d.setVar('WORKDIR', spdx_workdir) | ||
910 | # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). | ||
911 | d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) | ||
912 | |||
913 | # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the | ||
914 | # possibly requiring of the following tasks (such as some recipes's | ||
915 | # do_patch required 'B' existed). | ||
916 | bb.utils.mkdirhier(d.getVar('B')) | ||
917 | |||
918 | bb.build.exec_func('do_unpack', d) | ||
919 | # Copy source of kernel to spdx_workdir | ||
920 | if is_work_shared_spdx(d): | ||
921 | share_src = d.getVar('WORKDIR') | ||
922 | d.setVar('WORKDIR', spdx_workdir) | ||
923 | d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) | ||
924 | src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR') | ||
925 | bb.utils.mkdirhier(src_dir) | ||
926 | if bb.data.inherits_class('kernel',d): | ||
927 | share_src = d.getVar('STAGING_KERNEL_DIR') | ||
928 | cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/" | ||
929 | cmd_copy_shared_res = os.popen(cmd_copy_share).read() | ||
930 | bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res) | ||
931 | |||
932 | git_path = src_dir + "/.git" | ||
933 | if os.path.exists(git_path): | ||
934 | shutils.rmtree(git_path) | ||
935 | |||
936 | # Make sure gcc and kernel sources are patched only once | ||
937 | if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)): | ||
938 | bb.build.exec_func('do_patch', d) | ||
939 | |||
940 | # Some userland has no source. | ||
941 | if not os.path.exists( spdx_workdir ): | ||
942 | bb.utils.mkdirhier(spdx_workdir) | ||
943 | finally: | ||
944 | d.setVar("WORKDIR", workdir) | ||
945 | |||
946 | spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR" | ||
947 | |||
948 | do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" | 738 | do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" |
949 | do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" | 739 | do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" |
950 | 740 | ||
@@ -1002,6 +792,7 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx | |||
1002 | import os | 792 | import os |
1003 | import oe.spdx | 793 | import oe.spdx |
1004 | import oe.sbom | 794 | import oe.sbom |
795 | import oe.spdx_common | ||
1005 | import io | 796 | import io |
1006 | import json | 797 | import json |
1007 | from datetime import timezone, datetime | 798 | from datetime import timezone, datetime |
@@ -1009,8 +800,10 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx | |||
1009 | import tarfile | 800 | import tarfile |
1010 | import bb.compress.zstd | 801 | import bb.compress.zstd |
1011 | 802 | ||
1012 | providers = collect_package_providers(d) | 803 | license_data = oe.spdx_common.load_spdx_license_data(d) |
1013 | package_archs = d.getVar("SSTATE_ARCHS").split() | 804 | |
805 | providers = oe.spdx_common.collect_package_providers(d) | ||
806 | package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split() | ||
1014 | package_archs.reverse() | 807 | package_archs.reverse() |
1015 | 808 | ||
1016 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | 809 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") |
@@ -1019,10 +812,10 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx | |||
1019 | 812 | ||
1020 | doc = oe.spdx.SPDXDocument() | 813 | doc = oe.spdx.SPDXDocument() |
1021 | doc.name = rootfs_name | 814 | doc.name = rootfs_name |
1022 | doc.documentNamespace = get_doc_namespace(d, doc) | 815 | doc.documentNamespace = get_namespace(d, doc.name) |
1023 | doc.creationInfo.created = creation_time | 816 | doc.creationInfo.created = creation_time |
1024 | doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." | 817 | doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." |
1025 | doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] | 818 | doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d) |
1026 | doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") | 819 | doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") |
1027 | doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) | 820 | doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) |
1028 | doc.creationInfo.creators.append("Person: N/A ()") | 821 | doc.creationInfo.creators.append("Person: N/A ()") |
@@ -1035,52 +828,53 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx | |||
1035 | 828 | ||
1036 | doc.packages.append(image) | 829 | doc.packages.append(image) |
1037 | 830 | ||
1038 | for name in sorted(packages.keys()): | 831 | if packages: |
1039 | if name not in providers: | 832 | for name in sorted(packages.keys()): |
1040 | bb.fatal("Unable to find SPDX provider for '%s'" % name) | 833 | if name not in providers: |
1041 | 834 | bb.fatal("Unable to find SPDX provider for '%s'" % name) | |
1042 | pkg_name, pkg_hashfn = providers[name] | ||
1043 | |||
1044 | pkg_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, pkg_name, pkg_hashfn) | ||
1045 | if not pkg_spdx_path: | ||
1046 | bb.fatal("No SPDX file found for package %s, %s" % (pkg_name, pkg_hashfn)) | ||
1047 | |||
1048 | pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) | ||
1049 | 835 | ||
1050 | for p in pkg_doc.packages: | 836 | pkg_name, pkg_hashfn = providers[name] |
1051 | if p.name == name: | ||
1052 | pkg_ref = oe.spdx.SPDXExternalDocumentRef() | ||
1053 | pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name | ||
1054 | pkg_ref.spdxDocument = pkg_doc.documentNamespace | ||
1055 | pkg_ref.checksum.algorithm = "SHA1" | ||
1056 | pkg_ref.checksum.checksumValue = pkg_doc_sha1 | ||
1057 | 837 | ||
1058 | doc.externalDocumentRefs.append(pkg_ref) | 838 | pkg_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, pkg_name, pkg_hashfn) |
1059 | doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) | 839 | if not pkg_spdx_path: |
1060 | break | 840 | bb.fatal("No SPDX file found for package %s, %s" % (pkg_name, pkg_hashfn)) |
1061 | else: | ||
1062 | bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) | ||
1063 | 841 | ||
1064 | runtime_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "runtime-" + name, pkg_hashfn) | 842 | pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) |
1065 | if not runtime_spdx_path: | ||
1066 | bb.fatal("No runtime SPDX document found for %s, %s" % (name, pkg_hashfn)) | ||
1067 | 843 | ||
1068 | runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) | 844 | for p in pkg_doc.packages: |
845 | if p.name == name: | ||
846 | pkg_ref = oe.spdx.SPDXExternalDocumentRef() | ||
847 | pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name | ||
848 | pkg_ref.spdxDocument = pkg_doc.documentNamespace | ||
849 | pkg_ref.checksum.algorithm = "SHA1" | ||
850 | pkg_ref.checksum.checksumValue = pkg_doc_sha1 | ||
1069 | 851 | ||
1070 | runtime_ref = oe.spdx.SPDXExternalDocumentRef() | 852 | doc.externalDocumentRefs.append(pkg_ref) |
1071 | runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name | 853 | doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) |
1072 | runtime_ref.spdxDocument = runtime_doc.documentNamespace | 854 | break |
1073 | runtime_ref.checksum.algorithm = "SHA1" | 855 | else: |
1074 | runtime_ref.checksum.checksumValue = runtime_doc_sha1 | 856 | bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) |
1075 | 857 | ||
1076 | # "OTHER" isn't ideal here, but I can't find a relationship that makes sense | 858 | runtime_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "runtime-" + name, pkg_hashfn) |
1077 | doc.externalDocumentRefs.append(runtime_ref) | 859 | if not runtime_spdx_path: |
1078 | doc.add_relationship( | 860 | bb.fatal("No runtime SPDX document found for %s, %s" % (name, pkg_hashfn)) |
1079 | image, | 861 | |
1080 | "OTHER", | 862 | runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) |
1081 | "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), | 863 | |
1082 | comment="Runtime dependencies for %s" % name | 864 | runtime_ref = oe.spdx.SPDXExternalDocumentRef() |
1083 | ) | 865 | runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name |
866 | runtime_ref.spdxDocument = runtime_doc.documentNamespace | ||
867 | runtime_ref.checksum.algorithm = "SHA1" | ||
868 | runtime_ref.checksum.checksumValue = runtime_doc_sha1 | ||
869 | |||
870 | # "OTHER" isn't ideal here, but I can't find a relationship that makes sense | ||
871 | doc.externalDocumentRefs.append(runtime_ref) | ||
872 | doc.add_relationship( | ||
873 | image, | ||
874 | "OTHER", | ||
875 | "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), | ||
876 | comment="Runtime dependencies for %s" % name | ||
877 | ) | ||
1084 | bb.utils.mkdirhier(spdx_workdir) | 878 | bb.utils.mkdirhier(spdx_workdir) |
1085 | image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json") | 879 | image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json") |
1086 | 880 | ||
@@ -1161,4 +955,4 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx | |||
1161 | 955 | ||
1162 | tar.addfile(info, fileobj=index_str) | 956 | tar.addfile(info, fileobj=index_str) |
1163 | 957 | ||
1164 | combine_spdx[vardepsexclude] += "BB_NUMBER_THREADS SSTATE_ARCHS" | 958 | combine_spdx[vardepsexclude] += "BB_NUMBER_THREADS SPDX_MULTILIB_SSTATE_ARCHS" |