summaryrefslogtreecommitdiffstats
path: root/meta/classes/create-spdx-2.2.bbclass
diff options
context:
space:
mode:
Diffstat (limited to 'meta/classes/create-spdx-2.2.bbclass')
-rw-r--r--meta/classes/create-spdx-2.2.bbclass486
1 files changed, 149 insertions, 337 deletions
diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass
index 486efadba9..94e0108815 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -4,36 +4,9 @@
4# SPDX-License-Identifier: GPL-2.0-only 4# SPDX-License-Identifier: GPL-2.0-only
5# 5#
6 6
7DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx" 7inherit spdx-common
8 8
9# The product name that the CVE database uses. Defaults to BPN, but may need to 9SPDX_VERSION = "2.2"
10# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff).
11CVE_PRODUCT ??= "${BPN}"
12CVE_VERSION ??= "${PV}"
13
14SPDXDIR ??= "${WORKDIR}/spdx"
15SPDXDEPLOY = "${SPDXDIR}/deploy"
16SPDXWORK = "${SPDXDIR}/work"
17SPDXIMAGEWORK = "${SPDXDIR}/image-work"
18SPDXSDKWORK = "${SPDXDIR}/sdk-work"
19SPDXDEPS = "${SPDXDIR}/deps.json"
20
21SPDX_TOOL_NAME ??= "oe-spdx-creator"
22SPDX_TOOL_VERSION ??= "1.0"
23
24SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy"
25
26SPDX_INCLUDE_SOURCES ??= "0"
27SPDX_ARCHIVE_SOURCES ??= "0"
28SPDX_ARCHIVE_PACKAGED ??= "0"
29
30SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org"
31SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc"
32SPDX_PRETTY ??= "0"
33
34SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
35
36SPDX_CUSTOM_ANNOTATION_VARS ??= ""
37 10
38SPDX_ORG ??= "OpenEmbedded ()" 11SPDX_ORG ??= "OpenEmbedded ()"
39SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" 12SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
@@ -42,27 +15,16 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f
42 is the contact information for the person or organization who is doing the \ 15 is the contact information for the person or organization who is doing the \
43 build." 16 build."
44 17
45def extract_licenses(filename): 18SPDX_ARCHIVE_SOURCES ??= "0"
46 import re 19SPDX_ARCHIVE_PACKAGED ??= "0"
47
48 lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
49
50 try:
51 with open(filename, 'rb') as f:
52 size = min(15000, os.stat(filename).st_size)
53 txt = f.read(size)
54 licenses = re.findall(lic_regex, txt)
55 if licenses:
56 ascii_licenses = [lic.decode('ascii') for lic in licenses]
57 return ascii_licenses
58 except Exception as e:
59 bb.warn(f"Exception reading {filename}: {e}")
60 return None
61 20
62def get_doc_namespace(d, doc): 21def get_namespace(d, name):
63 import uuid 22 import uuid
64 namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) 23 namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
65 return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) 24 return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), name, str(uuid.uuid5(namespace_uuid, name)))
25
26SPDX_PACKAGE_VERSION ??= "${PV}"
27SPDX_PACKAGE_VERSION[doc] = "The version of a package, versionInfo in recipe, package and image"
66 28
67def create_annotation(d, comment): 29def create_annotation(d, comment):
68 from datetime import datetime, timezone 30 from datetime import datetime, timezone
@@ -80,31 +42,16 @@ def recipe_spdx_is_native(d, recipe):
80 a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and 42 a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and
81 a.comment == "isNative" for a in recipe.annotations) 43 a.comment == "isNative" for a in recipe.annotations)
82 44
83def is_work_shared_spdx(d):
84 return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR'))
85
86def get_json_indent(d): 45def get_json_indent(d):
87 if d.getVar("SPDX_PRETTY") == "1": 46 if d.getVar("SPDX_PRETTY") == "1":
88 return 2 47 return 2
89 return None 48 return None
90 49
91python() {
92 import json
93 if d.getVar("SPDX_LICENSE_DATA"):
94 return
95
96 with open(d.getVar("SPDX_LICENSES"), "r") as f:
97 data = json.load(f)
98 # Transform the license array to a dictionary
99 data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
100 d.setVar("SPDX_LICENSE_DATA", data)
101}
102 50
103def convert_license_to_spdx(lic, document, d, existing={}): 51def convert_license_to_spdx(lic, license_data, document, d, existing={}):
104 from pathlib import Path 52 from pathlib import Path
105 import oe.spdx 53 import oe.spdx
106 54
107 license_data = d.getVar("SPDX_LICENSE_DATA")
108 extracted = {} 55 extracted = {}
109 56
110 def add_extracted_license(ident, name): 57 def add_extracted_license(ident, name):
@@ -132,11 +79,17 @@ def convert_license_to_spdx(lic, document, d, existing={}):
132 pass 79 pass
133 if extracted_info.extractedText is None: 80 if extracted_info.extractedText is None:
134 # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set 81 # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set
135 filename = d.getVarFlag('NO_GENERIC_LICENSE', name) 82 entry = d.getVarFlag('NO_GENERIC_LICENSE', name).split(';')
83 filename = entry[0]
84 params = {i.split('=')[0]: i.split('=')[1] for i in entry[1:] if '=' in i}
85 beginline = int(params.get('beginline', 1))
86 endline = params.get('endline', None)
87 if endline:
88 endline = int(endline)
136 if filename: 89 if filename:
137 filename = d.expand("${S}/" + filename) 90 filename = d.expand("${S}/" + filename)
138 with open(filename, errors="replace") as f: 91 with open(filename, errors="replace") as f:
139 extracted_info.extractedText = f.read() 92 extracted_info.extractedText = "".join(line for idx, line in enumerate(f, 1) if beginline <= idx and idx <= (endline or idx))
140 else: 93 else:
141 bb.fatal("Cannot find any text for license %s" % name) 94 bb.fatal("Cannot find any text for license %s" % name)
142 95
@@ -172,37 +125,10 @@ def convert_license_to_spdx(lic, document, d, existing={}):
172 125
173 return ' '.join(convert(l) for l in lic_split) 126 return ' '.join(convert(l) for l in lic_split)
174 127
175def process_sources(d):
176 pn = d.getVar('PN')
177 assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
178 if pn in assume_provided:
179 for p in d.getVar("PROVIDES").split():
180 if p != pn:
181 pn = p
182 break
183
184 # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
185 # so avoid archiving source here.
186 if pn.startswith('glibc-locale'):
187 return False
188 if d.getVar('PN') == "libtool-cross":
189 return False
190 if d.getVar('PN') == "libgcc-initial":
191 return False
192 if d.getVar('PN') == "shadow-sysroot":
193 return False
194
195 # We just archive gcc-source for all the gcc related recipes
196 if d.getVar('BPN') in ['gcc', 'libgcc']:
197 bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
198 return False
199
200 return True
201
202
203def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): 128def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]):
204 from pathlib import Path 129 from pathlib import Path
205 import oe.spdx 130 import oe.spdx
131 import oe.spdx_common
206 import hashlib 132 import hashlib
207 133
208 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") 134 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
@@ -213,6 +139,11 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv
213 spdx_files = [] 139 spdx_files = []
214 140
215 file_counter = 1 141 file_counter = 1
142
143 check_compiled_sources = d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1"
144 if check_compiled_sources:
145 compiled_sources, types = oe.spdx_common.get_compiled_sources(d)
146 bb.debug(1, f"Total compiled files: {len(compiled_sources)}")
216 for subdir, dirs, files in os.walk(topdir): 147 for subdir, dirs, files in os.walk(topdir):
217 dirs[:] = [d for d in dirs if d not in ignore_dirs] 148 dirs[:] = [d for d in dirs if d not in ignore_dirs]
218 if subdir == str(topdir): 149 if subdir == str(topdir):
@@ -223,6 +154,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv
223 filename = str(filepath.relative_to(topdir)) 154 filename = str(filepath.relative_to(topdir))
224 155
225 if not filepath.is_symlink() and filepath.is_file(): 156 if not filepath.is_symlink() and filepath.is_file():
157 # Check if file is compiled
158 if check_compiled_sources:
159 if not oe.spdx_common.is_compiled_source(filename, compiled_sources, types):
160 continue
226 spdx_file = oe.spdx.SPDXFile() 161 spdx_file = oe.spdx.SPDXFile()
227 spdx_file.SPDXID = get_spdxid(file_counter) 162 spdx_file.SPDXID = get_spdxid(file_counter)
228 for t in get_types(filepath): 163 for t in get_types(filepath):
@@ -255,7 +190,7 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv
255 )) 190 ))
256 191
257 if "SOURCE" in spdx_file.fileTypes: 192 if "SOURCE" in spdx_file.fileTypes:
258 extracted_lics = extract_licenses(filepath) 193 extracted_lics = oe.spdx_common.extract_licenses(filepath)
259 if extracted_lics: 194 if extracted_lics:
260 spdx_file.licenseInfoInFiles = extracted_lics 195 spdx_file.licenseInfoInFiles = extracted_lics
261 196
@@ -313,7 +248,8 @@ def add_package_sources_from_debug(d, package_doc, spdx_package, package, packag
313 debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') 248 debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '')
314 else: 249 else:
315 debugsrc_path = search / debugsrc.lstrip("/") 250 debugsrc_path = search / debugsrc.lstrip("/")
316 if not debugsrc_path.exists(): 251 # We can only hash files below, skip directories, links, etc.
252 if not os.path.isfile(debugsrc_path):
317 continue 253 continue
318 254
319 file_sha256 = bb.utils.sha256_file(debugsrc_path) 255 file_sha256 = bb.utils.sha256_file(debugsrc_path)
@@ -346,26 +282,31 @@ def collect_dep_recipes(d, doc, spdx_recipe):
346 from pathlib import Path 282 from pathlib import Path
347 import oe.sbom 283 import oe.sbom
348 import oe.spdx 284 import oe.spdx
285 import oe.spdx_common
349 286
350 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 287 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
351 spdx_deps_file = Path(d.getVar("SPDXDEPS")) 288 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split()
352 package_archs = d.getVar("SSTATE_ARCHS").split()
353 package_archs.reverse() 289 package_archs.reverse()
354 290
355 dep_recipes = [] 291 dep_recipes = []
356 292
357 with spdx_deps_file.open("r") as f: 293 deps = oe.spdx_common.get_spdx_deps(d)
358 deps = json.load(f)
359 294
360 for dep_pn, dep_hashfn in deps: 295 for dep in deps:
361 dep_recipe_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "recipe-" + dep_pn, dep_hashfn) 296 # If this dependency is not calculated in the taskhash skip it.
297 # Otherwise, it can result in broken links since this task won't
298 # rebuild and see the new SPDX ID if the dependency changes
299 if not dep.in_taskhash:
300 continue
301
302 dep_recipe_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "recipe-" + dep.pn, dep.hashfn)
362 if not dep_recipe_path: 303 if not dep_recipe_path:
363 bb.fatal("Cannot find any SPDX file for recipe %s, %s" % (dep_pn, dep_hashfn)) 304 bb.fatal("Cannot find any SPDX file for recipe %s, %s" % (dep.pn, dep.hashfn))
364 305
365 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) 306 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path)
366 307
367 for pkg in spdx_dep_doc.packages: 308 for pkg in spdx_dep_doc.packages:
368 if pkg.name == dep_pn: 309 if pkg.name == dep.pn:
369 spdx_dep_recipe = pkg 310 spdx_dep_recipe = pkg
370 break 311 break
371 else: 312 else:
@@ -389,7 +330,7 @@ def collect_dep_recipes(d, doc, spdx_recipe):
389 330
390 return dep_recipes 331 return dep_recipes
391 332
392collect_dep_recipes[vardepsexclude] = "SSTATE_ARCHS" 333collect_dep_recipes[vardepsexclude] = "SPDX_MULTILIB_SSTATE_ARCHS"
393 334
394def collect_dep_sources(d, dep_recipes): 335def collect_dep_sources(d, dep_recipes):
395 import oe.sbom 336 import oe.sbom
@@ -424,99 +365,52 @@ def add_download_packages(d, doc, recipe):
424 for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()): 365 for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()):
425 f = bb.fetch2.FetchData(src_uri, d) 366 f = bb.fetch2.FetchData(src_uri, d)
426 367
427 for name in f.names: 368 package = oe.spdx.SPDXPackage()
428 package = oe.spdx.SPDXPackage() 369 package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1)
429 package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1) 370 package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1)
430 package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1)
431 371
432 if f.type == "file": 372 if f.type == "file":
433 continue 373 continue
434 374
435 uri = f.type 375 if f.method.supports_checksum(f):
436 proto = getattr(f, "proto", None) 376 for checksum_id in CHECKSUM_LIST:
437 if proto is not None: 377 if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS:
438 uri = uri + "+" + proto 378 continue
439 uri = uri + "://" + f.host + f.path
440
441 if f.method.supports_srcrev():
442 uri = uri + "@" + f.revisions[name]
443
444 if f.method.supports_checksum(f):
445 for checksum_id in CHECKSUM_LIST:
446 if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS:
447 continue
448
449 expected_checksum = getattr(f, "%s_expected" % checksum_id)
450 if expected_checksum is None:
451 continue
452
453 c = oe.spdx.SPDXChecksum()
454 c.algorithm = checksum_id.upper()
455 c.checksumValue = expected_checksum
456 package.checksums.append(c)
457
458 package.downloadLocation = uri
459 doc.packages.append(package)
460 doc.add_relationship(doc, "DESCRIBES", package)
461 # In the future, we might be able to do more fancy dependencies,
462 # but this should be sufficient for now
463 doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe)
464
465def collect_direct_deps(d, dep_task):
466 current_task = "do_" + d.getVar("BB_CURRENTTASK")
467 pn = d.getVar("PN")
468
469 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
470
471 for this_dep in taskdepdata.values():
472 if this_dep[0] == pn and this_dep[1] == current_task:
473 break
474 else:
475 bb.fatal(f"Unable to find this {pn}:{current_task} in taskdepdata")
476
477 deps = set()
478 for dep_name in this_dep[3]:
479 dep_data = taskdepdata[dep_name]
480 if dep_data[1] == dep_task and dep_data[0] != pn:
481 deps.add((dep_data[0], dep_data[7]))
482
483 return sorted(deps)
484
485collect_direct_deps[vardepsexclude] += "BB_TASKDEPDATA"
486collect_direct_deps[vardeps] += "DEPENDS"
487
488python do_collect_spdx_deps() {
489 # This task calculates the build time dependencies of the recipe, and is
490 # required because while a task can deptask on itself, those dependencies
491 # do not show up in BB_TASKDEPDATA. To work around that, this task does the
492 # deptask on do_create_spdx and writes out the dependencies it finds, then
493 # do_create_spdx reads in the found dependencies when writing the actual
494 # SPDX document
495 import json
496 from pathlib import Path
497 379
498 spdx_deps_file = Path(d.getVar("SPDXDEPS")) 380 expected_checksum = getattr(f, "%s_expected" % checksum_id)
381 if expected_checksum is None:
382 continue
499 383
500 deps = collect_direct_deps(d, "do_create_spdx") 384 c = oe.spdx.SPDXChecksum()
385 c.algorithm = checksum_id.upper()
386 c.checksumValue = expected_checksum
387 package.checksums.append(c)
388
389 package.downloadLocation = oe.spdx_common.fetch_data_to_uri(f, f.name)
390 doc.packages.append(package)
391 doc.add_relationship(doc, "DESCRIBES", package)
392 # In the future, we might be able to do more fancy dependencies,
393 # but this should be sufficient for now
394 doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe)
395
396def get_license_list_version(license_data, d):
397 # Newer versions of the SPDX license list are SemVer ("MAJOR.MINOR.MICRO"),
398 # but SPDX 2 only uses "MAJOR.MINOR".
399 return ".".join(license_data["licenseListVersion"].split(".")[:2])
501 400
502 with spdx_deps_file.open("w") as f:
503 json.dump(deps, f)
504}
505# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
506addtask do_collect_spdx_deps after do_unpack
507do_collect_spdx_deps[depends] += "${PATCHDEPENDENCY}"
508do_collect_spdx_deps[deptask] = "do_create_spdx"
509do_collect_spdx_deps[dirs] = "${SPDXDIR}"
510 401
511python do_create_spdx() { 402python do_create_spdx() {
512 from datetime import datetime, timezone 403 from datetime import datetime, timezone
513 import oe.sbom 404 import oe.sbom
514 import oe.spdx 405 import oe.spdx
406 import oe.spdx_common
515 import uuid 407 import uuid
516 from pathlib import Path 408 from pathlib import Path
517 from contextlib import contextmanager 409 from contextlib import contextmanager
518 import oe.cve_check 410 import oe.cve_check
519 411
412 license_data = oe.spdx_common.load_spdx_license_data(d)
413
520 @contextmanager 414 @contextmanager
521 def optional_tarfile(name, guard, mode="w"): 415 def optional_tarfile(name, guard, mode="w"):
522 import tarfile 416 import tarfile
@@ -545,17 +439,17 @@ python do_create_spdx() {
545 doc = oe.spdx.SPDXDocument() 439 doc = oe.spdx.SPDXDocument()
546 440
547 doc.name = "recipe-" + d.getVar("PN") 441 doc.name = "recipe-" + d.getVar("PN")
548 doc.documentNamespace = get_doc_namespace(d, doc) 442 doc.documentNamespace = get_namespace(d, doc.name)
549 doc.creationInfo.created = creation_time 443 doc.creationInfo.created = creation_time
550 doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." 444 doc.creationInfo.comment = "This document was created by analyzing recipe files during the build."
551 doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] 445 doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d)
552 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 446 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
553 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 447 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
554 doc.creationInfo.creators.append("Person: N/A ()") 448 doc.creationInfo.creators.append("Person: N/A ()")
555 449
556 recipe = oe.spdx.SPDXPackage() 450 recipe = oe.spdx.SPDXPackage()
557 recipe.name = d.getVar("PN") 451 recipe.name = d.getVar("PN")
558 recipe.versionInfo = d.getVar("PV") 452 recipe.versionInfo = d.getVar("SPDX_PACKAGE_VERSION")
559 recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) 453 recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
560 recipe.supplier = d.getVar("SPDX_SUPPLIER") 454 recipe.supplier = d.getVar("SPDX_SUPPLIER")
561 if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): 455 if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d):
@@ -567,7 +461,7 @@ python do_create_spdx() {
567 461
568 license = d.getVar("LICENSE") 462 license = d.getVar("LICENSE")
569 if license: 463 if license:
570 recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) 464 recipe.licenseDeclared = convert_license_to_spdx(license, license_data, doc, d)
571 465
572 summary = d.getVar("SUMMARY") 466 summary = d.getVar("SUMMARY")
573 if summary: 467 if summary:
@@ -604,10 +498,10 @@ python do_create_spdx() {
604 498
605 add_download_packages(d, doc, recipe) 499 add_download_packages(d, doc, recipe)
606 500
607 if process_sources(d) and include_sources: 501 if oe.spdx_common.process_sources(d) and include_sources:
608 recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") 502 recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst")
609 with optional_tarfile(recipe_archive, archive_sources) as archive: 503 with optional_tarfile(recipe_archive, archive_sources) as archive:
610 spdx_get_src(d) 504 oe.spdx_common.get_patched_src(d)
611 505
612 add_package_files( 506 add_package_files(
613 d, 507 d,
@@ -649,10 +543,10 @@ python do_create_spdx() {
649 package_doc = oe.spdx.SPDXDocument() 543 package_doc = oe.spdx.SPDXDocument()
650 pkg_name = d.getVar("PKG:%s" % package) or package 544 pkg_name = d.getVar("PKG:%s" % package) or package
651 package_doc.name = pkg_name 545 package_doc.name = pkg_name
652 package_doc.documentNamespace = get_doc_namespace(d, package_doc) 546 package_doc.documentNamespace = get_namespace(d, package_doc.name)
653 package_doc.creationInfo.created = creation_time 547 package_doc.creationInfo.created = creation_time
654 package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." 548 package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build."
655 package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] 549 package_doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d)
656 package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 550 package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
657 package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 551 package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
658 package_doc.creationInfo.creators.append("Person: N/A ()") 552 package_doc.creationInfo.creators.append("Person: N/A ()")
@@ -664,8 +558,8 @@ python do_create_spdx() {
664 558
665 spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) 559 spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name)
666 spdx_package.name = pkg_name 560 spdx_package.name = pkg_name
667 spdx_package.versionInfo = d.getVar("PV") 561 spdx_package.versionInfo = d.getVar("SPDX_PACKAGE_VERSION")
668 spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) 562 spdx_package.licenseDeclared = convert_license_to_spdx(package_license, license_data, package_doc, d, found_licenses)
669 spdx_package.supplier = d.getVar("SPDX_SUPPLIER") 563 spdx_package.supplier = d.getVar("SPDX_SUPPLIER")
670 564
671 package_doc.packages.append(spdx_package) 565 package_doc.packages.append(spdx_package)
@@ -708,50 +602,16 @@ addtask do_create_spdx_setscene
708 602
709do_create_spdx[dirs] = "${SPDXWORK}" 603do_create_spdx[dirs] = "${SPDXWORK}"
710do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" 604do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}"
711do_create_spdx[depends] += "${PATCHDEPENDENCY}" 605do_create_spdx[depends] += " \
712 606 ${PATCHDEPENDENCY} \
713def collect_package_providers(d): 607 ${@create_spdx_source_deps(d)} \
714 from pathlib import Path 608"
715 import oe.sbom
716 import oe.spdx
717 import json
718
719 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
720
721 providers = {}
722
723 deps = collect_direct_deps(d, "do_create_spdx")
724 deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME")))
725
726 for dep_pn, dep_hashfn in deps:
727 localdata = d
728 recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
729 if not recipe_data:
730 localdata = bb.data.createCopy(d)
731 localdata.setVar("PKGDATA_DIR", "${PKGDATA_DIR_SDK}")
732 recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
733
734 for pkg in recipe_data.get("PACKAGES", "").split():
735
736 pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, localdata)
737 rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items())
738 rprovides.add(pkg)
739
740 if "PKG" in pkg_data:
741 pkg = pkg_data["PKG"]
742 rprovides.add(pkg)
743
744 for r in rprovides:
745 providers[r] = (pkg, dep_hashfn)
746
747 return providers
748
749collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA"
750 609
751python do_create_runtime_spdx() { 610python do_create_runtime_spdx() {
752 from datetime import datetime, timezone 611 from datetime import datetime, timezone
753 import oe.sbom 612 import oe.sbom
754 import oe.spdx 613 import oe.spdx
614 import oe.spdx_common
755 import oe.packagedata 615 import oe.packagedata
756 from pathlib import Path 616 from pathlib import Path
757 617
@@ -761,9 +621,11 @@ python do_create_runtime_spdx() {
761 621
762 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 622 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
763 623
764 providers = collect_package_providers(d) 624 license_data = oe.spdx_common.load_spdx_license_data(d)
625
626 providers = oe.spdx_common.collect_package_providers(d)
765 pkg_arch = d.getVar("SSTATE_PKGARCH") 627 pkg_arch = d.getVar("SSTATE_PKGARCH")
766 package_archs = d.getVar("SSTATE_ARCHS").split() 628 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split()
767 package_archs.reverse() 629 package_archs.reverse()
768 630
769 if not is_native: 631 if not is_native:
@@ -794,10 +656,10 @@ python do_create_runtime_spdx() {
794 656
795 runtime_doc = oe.spdx.SPDXDocument() 657 runtime_doc = oe.spdx.SPDXDocument()
796 runtime_doc.name = "runtime-" + pkg_name 658 runtime_doc.name = "runtime-" + pkg_name
797 runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) 659 runtime_doc.documentNamespace = get_namespace(localdata, runtime_doc.name)
798 runtime_doc.creationInfo.created = creation_time 660 runtime_doc.creationInfo.created = creation_time
799 runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." 661 runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies."
800 runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] 662 runtime_doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d)
801 runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 663 runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
802 runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 664 runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
803 runtime_doc.creationInfo.creators.append("Person: N/A ()") 665 runtime_doc.creationInfo.creators.append("Person: N/A ()")
@@ -869,7 +731,7 @@ python do_create_runtime_spdx() {
869 oe.sbom.write_doc(d, runtime_doc, pkg_arch, "runtime", spdx_deploy, indent=get_json_indent(d)) 731 oe.sbom.write_doc(d, runtime_doc, pkg_arch, "runtime", spdx_deploy, indent=get_json_indent(d))
870} 732}
871 733
872do_create_runtime_spdx[vardepsexclude] += "OVERRIDES SSTATE_ARCHS" 734do_create_runtime_spdx[vardepsexclude] += "OVERRIDES SPDX_MULTILIB_SSTATE_ARCHS"
873 735
874addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work 736addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work
875SSTATETASKS += "do_create_runtime_spdx" 737SSTATETASKS += "do_create_runtime_spdx"
@@ -885,60 +747,6 @@ do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}"
885do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" 747do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}"
886do_create_runtime_spdx[rdeptask] = "do_create_spdx" 748do_create_runtime_spdx[rdeptask] = "do_create_spdx"
887 749
888def spdx_get_src(d):
889 """
890 save patched source of the recipe in SPDX_WORKDIR.
891 """
892 import shutil
893 spdx_workdir = d.getVar('SPDXWORK')
894 spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE')
895 pn = d.getVar('PN')
896
897 workdir = d.getVar("WORKDIR")
898
899 try:
900 # The kernel class functions require it to be on work-shared, so we dont change WORKDIR
901 if not is_work_shared_spdx(d):
902 # Change the WORKDIR to make do_unpack do_patch run in another dir.
903 d.setVar('WORKDIR', spdx_workdir)
904 # Restore the original path to recipe's native sysroot (it's relative to WORKDIR).
905 d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)
906
907 # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the
908 # possibly requiring of the following tasks (such as some recipes's
909 # do_patch required 'B' existed).
910 bb.utils.mkdirhier(d.getVar('B'))
911
912 bb.build.exec_func('do_unpack', d)
913 # Copy source of kernel to spdx_workdir
914 if is_work_shared_spdx(d):
915 share_src = d.getVar('WORKDIR')
916 d.setVar('WORKDIR', spdx_workdir)
917 d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)
918 src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR')
919 bb.utils.mkdirhier(src_dir)
920 if bb.data.inherits_class('kernel',d):
921 share_src = d.getVar('STAGING_KERNEL_DIR')
922 cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/"
923 cmd_copy_shared_res = os.popen(cmd_copy_share).read()
924 bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res)
925
926 git_path = src_dir + "/.git"
927 if os.path.exists(git_path):
928 shutils.rmtree(git_path)
929
930 # Make sure gcc and kernel sources are patched only once
931 if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)):
932 bb.build.exec_func('do_patch', d)
933
934 # Some userland has no source.
935 if not os.path.exists( spdx_workdir ):
936 bb.utils.mkdirhier(spdx_workdir)
937 finally:
938 d.setVar("WORKDIR", workdir)
939
940spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR"
941
942do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" 750do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
943do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" 751do_rootfs[cleandirs] += "${SPDXIMAGEWORK}"
944 752
@@ -996,6 +804,7 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
996 import os 804 import os
997 import oe.spdx 805 import oe.spdx
998 import oe.sbom 806 import oe.sbom
807 import oe.spdx_common
999 import io 808 import io
1000 import json 809 import json
1001 from datetime import timezone, datetime 810 from datetime import timezone, datetime
@@ -1003,8 +812,10 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
1003 import tarfile 812 import tarfile
1004 import bb.compress.zstd 813 import bb.compress.zstd
1005 814
1006 providers = collect_package_providers(d) 815 license_data = oe.spdx_common.load_spdx_license_data(d)
1007 package_archs = d.getVar("SSTATE_ARCHS").split() 816
817 providers = oe.spdx_common.collect_package_providers(d)
818 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split()
1008 package_archs.reverse() 819 package_archs.reverse()
1009 820
1010 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 821 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -1013,68 +824,69 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
1013 824
1014 doc = oe.spdx.SPDXDocument() 825 doc = oe.spdx.SPDXDocument()
1015 doc.name = rootfs_name 826 doc.name = rootfs_name
1016 doc.documentNamespace = get_doc_namespace(d, doc) 827 doc.documentNamespace = get_namespace(d, doc.name)
1017 doc.creationInfo.created = creation_time 828 doc.creationInfo.created = creation_time
1018 doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." 829 doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build."
1019 doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] 830 doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d)
1020 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 831 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
1021 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 832 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
1022 doc.creationInfo.creators.append("Person: N/A ()") 833 doc.creationInfo.creators.append("Person: N/A ()")
1023 834
1024 image = oe.spdx.SPDXPackage() 835 image = oe.spdx.SPDXPackage()
1025 image.name = d.getVar("PN") 836 image.name = d.getVar("PN")
1026 image.versionInfo = d.getVar("PV") 837 image.versionInfo = d.getVar("SPDX_PACKAGE_VERSION")
1027 image.SPDXID = rootfs_spdxid 838 image.SPDXID = rootfs_spdxid
1028 image.supplier = d.getVar("SPDX_SUPPLIER") 839 image.supplier = d.getVar("SPDX_SUPPLIER")
1029 840
1030 doc.packages.append(image) 841 doc.packages.append(image)
1031 842
1032 for name in sorted(packages.keys()): 843 if packages:
1033 if name not in providers: 844 for name in sorted(packages.keys()):
1034 bb.fatal("Unable to find SPDX provider for '%s'" % name) 845 if name not in providers:
1035 846 bb.fatal("Unable to find SPDX provider for '%s'" % name)
1036 pkg_name, pkg_hashfn = providers[name]
1037 847
1038 pkg_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, pkg_name, pkg_hashfn) 848 pkg_name, pkg_hashfn = providers[name]
1039 if not pkg_spdx_path:
1040 bb.fatal("No SPDX file found for package %s, %s" % (pkg_name, pkg_hashfn))
1041 849
1042 pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) 850 pkg_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, pkg_name, pkg_hashfn)
851 if not pkg_spdx_path:
852 bb.fatal("No SPDX file found for package %s, %s" % (pkg_name, pkg_hashfn))
1043 853
1044 for p in pkg_doc.packages: 854 pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path)
1045 if p.name == name:
1046 pkg_ref = oe.spdx.SPDXExternalDocumentRef()
1047 pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name
1048 pkg_ref.spdxDocument = pkg_doc.documentNamespace
1049 pkg_ref.checksum.algorithm = "SHA1"
1050 pkg_ref.checksum.checksumValue = pkg_doc_sha1
1051 855
1052 doc.externalDocumentRefs.append(pkg_ref) 856 for p in pkg_doc.packages:
1053 doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) 857 if p.name == name:
1054 break 858 pkg_ref = oe.spdx.SPDXExternalDocumentRef()
1055 else: 859 pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name
1056 bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) 860 pkg_ref.spdxDocument = pkg_doc.documentNamespace
861 pkg_ref.checksum.algorithm = "SHA1"
862 pkg_ref.checksum.checksumValue = pkg_doc_sha1
1057 863
1058 runtime_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "runtime-" + name, pkg_hashfn) 864 doc.externalDocumentRefs.append(pkg_ref)
1059 if not runtime_spdx_path: 865 doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID))
1060 bb.fatal("No runtime SPDX document found for %s, %s" % (name, pkg_hashfn)) 866 break
1061 867 else:
1062 runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) 868 bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path))
1063 869
1064 runtime_ref = oe.spdx.SPDXExternalDocumentRef() 870 runtime_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "runtime-" + name, pkg_hashfn)
1065 runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name 871 if not runtime_spdx_path:
1066 runtime_ref.spdxDocument = runtime_doc.documentNamespace 872 bb.fatal("No runtime SPDX document found for %s, %s" % (name, pkg_hashfn))
1067 runtime_ref.checksum.algorithm = "SHA1" 873
1068 runtime_ref.checksum.checksumValue = runtime_doc_sha1 874 runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path)
1069 875
1070 # "OTHER" isn't ideal here, but I can't find a relationship that makes sense 876 runtime_ref = oe.spdx.SPDXExternalDocumentRef()
1071 doc.externalDocumentRefs.append(runtime_ref) 877 runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name
1072 doc.add_relationship( 878 runtime_ref.spdxDocument = runtime_doc.documentNamespace
1073 image, 879 runtime_ref.checksum.algorithm = "SHA1"
1074 "OTHER", 880 runtime_ref.checksum.checksumValue = runtime_doc_sha1
1075 "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), 881
1076 comment="Runtime dependencies for %s" % name 882 # "OTHER" isn't ideal here, but I can't find a relationship that makes sense
1077 ) 883 doc.externalDocumentRefs.append(runtime_ref)
884 doc.add_relationship(
885 image,
886 "OTHER",
887 "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID),
888 comment="Runtime dependencies for %s" % name
889 )
1078 bb.utils.mkdirhier(spdx_workdir) 890 bb.utils.mkdirhier(spdx_workdir)
1079 image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json") 891 image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json")
1080 892
@@ -1155,4 +967,4 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
1155 967
1156 tar.addfile(info, fileobj=index_str) 968 tar.addfile(info, fileobj=index_str)
1157 969
1158combine_spdx[vardepsexclude] += "BB_NUMBER_THREADS SSTATE_ARCHS" 970combine_spdx[vardepsexclude] += "BB_NUMBER_THREADS SPDX_MULTILIB_SSTATE_ARCHS"