summaryrefslogtreecommitdiffstats
path: root/meta/classes/create-spdx-2.2.bbclass
diff options
context:
space:
mode:
Diffstat (limited to 'meta/classes/create-spdx-2.2.bbclass')
-rw-r--r--meta/classes/create-spdx-2.2.bbclass482
1 files changed, 144 insertions, 338 deletions
diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass
index 7c8a0b8b0f..94e0108815 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -4,36 +4,9 @@
4# SPDX-License-Identifier: GPL-2.0-only 4# SPDX-License-Identifier: GPL-2.0-only
5# 5#
6 6
7DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx" 7inherit spdx-common
8 8
9# The product name that the CVE database uses. Defaults to BPN, but may need to 9SPDX_VERSION = "2.2"
10# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff).
11CVE_PRODUCT ??= "${BPN}"
12CVE_VERSION ??= "${PV}"
13
14SPDXDIR ??= "${WORKDIR}/spdx"
15SPDXDEPLOY = "${SPDXDIR}/deploy"
16SPDXWORK = "${SPDXDIR}/work"
17SPDXIMAGEWORK = "${SPDXDIR}/image-work"
18SPDXSDKWORK = "${SPDXDIR}/sdk-work"
19SPDXDEPS = "${SPDXDIR}/deps.json"
20
21SPDX_TOOL_NAME ??= "oe-spdx-creator"
22SPDX_TOOL_VERSION ??= "1.0"
23
24SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy"
25
26SPDX_INCLUDE_SOURCES ??= "0"
27SPDX_ARCHIVE_SOURCES ??= "0"
28SPDX_ARCHIVE_PACKAGED ??= "0"
29
30SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org"
31SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdocs"
32SPDX_PRETTY ??= "0"
33
34SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
35
36SPDX_CUSTOM_ANNOTATION_VARS ??= ""
37 10
38SPDX_ORG ??= "OpenEmbedded ()" 11SPDX_ORG ??= "OpenEmbedded ()"
39SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" 12SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
@@ -42,27 +15,16 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created f
42 is the contact information for the person or organization who is doing the \ 15 is the contact information for the person or organization who is doing the \
43 build." 16 build."
44 17
45def extract_licenses(filename): 18SPDX_ARCHIVE_SOURCES ??= "0"
46 import re 19SPDX_ARCHIVE_PACKAGED ??= "0"
47
48 lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
49
50 try:
51 with open(filename, 'rb') as f:
52 size = min(15000, os.stat(filename).st_size)
53 txt = f.read(size)
54 licenses = re.findall(lic_regex, txt)
55 if licenses:
56 ascii_licenses = [lic.decode('ascii') for lic in licenses]
57 return ascii_licenses
58 except Exception as e:
59 bb.warn(f"Exception reading {filename}: {e}")
60 return None
61 20
62def get_doc_namespace(d, doc): 21def get_namespace(d, name):
63 import uuid 22 import uuid
64 namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) 23 namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
65 return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) 24 return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), name, str(uuid.uuid5(namespace_uuid, name)))
25
26SPDX_PACKAGE_VERSION ??= "${PV}"
27SPDX_PACKAGE_VERSION[doc] = "The version of a package, versionInfo in recipe, package and image"
66 28
67def create_annotation(d, comment): 29def create_annotation(d, comment):
68 from datetime import datetime, timezone 30 from datetime import datetime, timezone
@@ -80,31 +42,16 @@ def recipe_spdx_is_native(d, recipe):
80 a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and 42 a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and
81 a.comment == "isNative" for a in recipe.annotations) 43 a.comment == "isNative" for a in recipe.annotations)
82 44
83def is_work_shared_spdx(d):
84 return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR'))
85
86def get_json_indent(d): 45def get_json_indent(d):
87 if d.getVar("SPDX_PRETTY") == "1": 46 if d.getVar("SPDX_PRETTY") == "1":
88 return 2 47 return 2
89 return None 48 return None
90 49
91python() {
92 import json
93 if d.getVar("SPDX_LICENSE_DATA"):
94 return
95
96 with open(d.getVar("SPDX_LICENSES"), "r") as f:
97 data = json.load(f)
98 # Transform the license array to a dictionary
99 data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
100 d.setVar("SPDX_LICENSE_DATA", data)
101}
102 50
103def convert_license_to_spdx(lic, document, d, existing={}): 51def convert_license_to_spdx(lic, license_data, document, d, existing={}):
104 from pathlib import Path 52 from pathlib import Path
105 import oe.spdx 53 import oe.spdx
106 54
107 license_data = d.getVar("SPDX_LICENSE_DATA")
108 extracted = {} 55 extracted = {}
109 56
110 def add_extracted_license(ident, name): 57 def add_extracted_license(ident, name):
@@ -132,11 +79,17 @@ def convert_license_to_spdx(lic, document, d, existing={}):
132 pass 79 pass
133 if extracted_info.extractedText is None: 80 if extracted_info.extractedText is None:
134 # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set 81 # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set
135 filename = d.getVarFlag('NO_GENERIC_LICENSE', name) 82 entry = d.getVarFlag('NO_GENERIC_LICENSE', name).split(';')
83 filename = entry[0]
84 params = {i.split('=')[0]: i.split('=')[1] for i in entry[1:] if '=' in i}
85 beginline = int(params.get('beginline', 1))
86 endline = params.get('endline', None)
87 if endline:
88 endline = int(endline)
136 if filename: 89 if filename:
137 filename = d.expand("${S}/" + filename) 90 filename = d.expand("${S}/" + filename)
138 with open(filename, errors="replace") as f: 91 with open(filename, errors="replace") as f:
139 extracted_info.extractedText = f.read() 92 extracted_info.extractedText = "".join(line for idx, line in enumerate(f, 1) if beginline <= idx and idx <= (endline or idx))
140 else: 93 else:
141 bb.fatal("Cannot find any text for license %s" % name) 94 bb.fatal("Cannot find any text for license %s" % name)
142 95
@@ -172,37 +125,10 @@ def convert_license_to_spdx(lic, document, d, existing={}):
172 125
173 return ' '.join(convert(l) for l in lic_split) 126 return ' '.join(convert(l) for l in lic_split)
174 127
175def process_sources(d):
176 pn = d.getVar('PN')
177 assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
178 if pn in assume_provided:
179 for p in d.getVar("PROVIDES").split():
180 if p != pn:
181 pn = p
182 break
183
184 # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
185 # so avoid archiving source here.
186 if pn.startswith('glibc-locale'):
187 return False
188 if d.getVar('PN') == "libtool-cross":
189 return False
190 if d.getVar('PN') == "libgcc-initial":
191 return False
192 if d.getVar('PN') == "shadow-sysroot":
193 return False
194
195 # We just archive gcc-source for all the gcc related recipes
196 if d.getVar('BPN') in ['gcc', 'libgcc']:
197 bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
198 return False
199
200 return True
201
202
203def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): 128def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]):
204 from pathlib import Path 129 from pathlib import Path
205 import oe.spdx 130 import oe.spdx
131 import oe.spdx_common
206 import hashlib 132 import hashlib
207 133
208 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") 134 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
@@ -213,6 +139,11 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv
213 spdx_files = [] 139 spdx_files = []
214 140
215 file_counter = 1 141 file_counter = 1
142
143 check_compiled_sources = d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1"
144 if check_compiled_sources:
145 compiled_sources, types = oe.spdx_common.get_compiled_sources(d)
146 bb.debug(1, f"Total compiled files: {len(compiled_sources)}")
216 for subdir, dirs, files in os.walk(topdir): 147 for subdir, dirs, files in os.walk(topdir):
217 dirs[:] = [d for d in dirs if d not in ignore_dirs] 148 dirs[:] = [d for d in dirs if d not in ignore_dirs]
218 if subdir == str(topdir): 149 if subdir == str(topdir):
@@ -223,6 +154,10 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv
223 filename = str(filepath.relative_to(topdir)) 154 filename = str(filepath.relative_to(topdir))
224 155
225 if not filepath.is_symlink() and filepath.is_file(): 156 if not filepath.is_symlink() and filepath.is_file():
157 # Check if file is compiled
158 if check_compiled_sources:
159 if not oe.spdx_common.is_compiled_source(filename, compiled_sources, types):
160 continue
226 spdx_file = oe.spdx.SPDXFile() 161 spdx_file = oe.spdx.SPDXFile()
227 spdx_file.SPDXID = get_spdxid(file_counter) 162 spdx_file.SPDXID = get_spdxid(file_counter)
228 for t in get_types(filepath): 163 for t in get_types(filepath):
@@ -255,7 +190,7 @@ def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archiv
255 )) 190 ))
256 191
257 if "SOURCE" in spdx_file.fileTypes: 192 if "SOURCE" in spdx_file.fileTypes:
258 extracted_lics = extract_licenses(filepath) 193 extracted_lics = oe.spdx_common.extract_licenses(filepath)
259 if extracted_lics: 194 if extracted_lics:
260 spdx_file.licenseInfoInFiles = extracted_lics 195 spdx_file.licenseInfoInFiles = extracted_lics
261 196
@@ -313,7 +248,8 @@ def add_package_sources_from_debug(d, package_doc, spdx_package, package, packag
313 debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') 248 debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '')
314 else: 249 else:
315 debugsrc_path = search / debugsrc.lstrip("/") 250 debugsrc_path = search / debugsrc.lstrip("/")
316 if not debugsrc_path.exists(): 251 # We can only hash files below, skip directories, links, etc.
252 if not os.path.isfile(debugsrc_path):
317 continue 253 continue
318 254
319 file_sha256 = bb.utils.sha256_file(debugsrc_path) 255 file_sha256 = bb.utils.sha256_file(debugsrc_path)
@@ -346,32 +282,31 @@ def collect_dep_recipes(d, doc, spdx_recipe):
346 from pathlib import Path 282 from pathlib import Path
347 import oe.sbom 283 import oe.sbom
348 import oe.spdx 284 import oe.spdx
285 import oe.spdx_common
349 286
350 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) 287 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
351 spdx_deps_file = Path(d.getVar("SPDXDEPS")) 288 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split()
352 package_archs = d.getVar("SSTATE_ARCHS").split()
353 package_archs.reverse() 289 package_archs.reverse()
354 290
355 dep_recipes = [] 291 dep_recipes = []
356 292
357 with spdx_deps_file.open("r") as f: 293 deps = oe.spdx_common.get_spdx_deps(d)
358 deps = json.load(f)
359 294
360 for dep_pn, dep_hashfn, in_taskhash in deps: 295 for dep in deps:
361 # If this dependency is not calculated in the taskhash skip it. 296 # If this dependency is not calculated in the taskhash skip it.
362 # Otherwise, it can result in broken links since this task won't 297 # Otherwise, it can result in broken links since this task won't
363 # rebuild and see the new SPDX ID if the dependency changes 298 # rebuild and see the new SPDX ID if the dependency changes
364 if not in_taskhash: 299 if not dep.in_taskhash:
365 continue 300 continue
366 301
367 dep_recipe_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "recipe-" + dep_pn, dep_hashfn) 302 dep_recipe_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "recipe-" + dep.pn, dep.hashfn)
368 if not dep_recipe_path: 303 if not dep_recipe_path:
369 bb.fatal("Cannot find any SPDX file for recipe %s, %s" % (dep_pn, dep_hashfn)) 304 bb.fatal("Cannot find any SPDX file for recipe %s, %s" % (dep.pn, dep.hashfn))
370 305
371 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) 306 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path)
372 307
373 for pkg in spdx_dep_doc.packages: 308 for pkg in spdx_dep_doc.packages:
374 if pkg.name == dep_pn: 309 if pkg.name == dep.pn:
375 spdx_dep_recipe = pkg 310 spdx_dep_recipe = pkg
376 break 311 break
377 else: 312 else:
@@ -395,7 +330,7 @@ def collect_dep_recipes(d, doc, spdx_recipe):
395 330
396 return dep_recipes 331 return dep_recipes
397 332
398collect_dep_recipes[vardepsexclude] = "SSTATE_ARCHS" 333collect_dep_recipes[vardepsexclude] = "SPDX_MULTILIB_SSTATE_ARCHS"
399 334
400def collect_dep_sources(d, dep_recipes): 335def collect_dep_sources(d, dep_recipes):
401 import oe.sbom 336 import oe.sbom
@@ -430,99 +365,52 @@ def add_download_packages(d, doc, recipe):
430 for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()): 365 for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()):
431 f = bb.fetch2.FetchData(src_uri, d) 366 f = bb.fetch2.FetchData(src_uri, d)
432 367
433 for name in f.names: 368 package = oe.spdx.SPDXPackage()
434 package = oe.spdx.SPDXPackage() 369 package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1)
435 package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1) 370 package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1)
436 package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1)
437 371
438 if f.type == "file": 372 if f.type == "file":
439 continue 373 continue
440 374
441 uri = f.type 375 if f.method.supports_checksum(f):
442 proto = getattr(f, "proto", None) 376 for checksum_id in CHECKSUM_LIST:
443 if proto is not None: 377 if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS:
444 uri = uri + "+" + proto 378 continue
445 uri = uri + "://" + f.host + f.path
446
447 if f.method.supports_srcrev():
448 uri = uri + "@" + f.revisions[name]
449
450 if f.method.supports_checksum(f):
451 for checksum_id in CHECKSUM_LIST:
452 if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS:
453 continue
454
455 expected_checksum = getattr(f, "%s_expected" % checksum_id)
456 if expected_checksum is None:
457 continue
458
459 c = oe.spdx.SPDXChecksum()
460 c.algorithm = checksum_id.upper()
461 c.checksumValue = expected_checksum
462 package.checksums.append(c)
463
464 package.downloadLocation = uri
465 doc.packages.append(package)
466 doc.add_relationship(doc, "DESCRIBES", package)
467 # In the future, we might be able to do more fancy dependencies,
468 # but this should be sufficient for now
469 doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe)
470
471def collect_direct_deps(d, dep_task):
472 current_task = "do_" + d.getVar("BB_CURRENTTASK")
473 pn = d.getVar("PN")
474
475 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
476
477 for this_dep in taskdepdata.values():
478 if this_dep[0] == pn and this_dep[1] == current_task:
479 break
480 else:
481 bb.fatal(f"Unable to find this {pn}:{current_task} in taskdepdata")
482
483 deps = set()
484 for dep_name in this_dep[3]:
485 dep_data = taskdepdata[dep_name]
486 if dep_data[1] == dep_task and dep_data[0] != pn:
487 deps.add((dep_data[0], dep_data[7], dep_name in this_dep[8]))
488
489 return sorted(deps)
490
491collect_direct_deps[vardepsexclude] += "BB_TASKDEPDATA"
492collect_direct_deps[vardeps] += "DEPENDS"
493
494python do_collect_spdx_deps() {
495 # This task calculates the build time dependencies of the recipe, and is
496 # required because while a task can deptask on itself, those dependencies
497 # do not show up in BB_TASKDEPDATA. To work around that, this task does the
498 # deptask on do_create_spdx and writes out the dependencies it finds, then
499 # do_create_spdx reads in the found dependencies when writing the actual
500 # SPDX document
501 import json
502 from pathlib import Path
503 379
504 spdx_deps_file = Path(d.getVar("SPDXDEPS")) 380 expected_checksum = getattr(f, "%s_expected" % checksum_id)
381 if expected_checksum is None:
382 continue
505 383
506 deps = collect_direct_deps(d, "do_create_spdx") 384 c = oe.spdx.SPDXChecksum()
385 c.algorithm = checksum_id.upper()
386 c.checksumValue = expected_checksum
387 package.checksums.append(c)
388
389 package.downloadLocation = oe.spdx_common.fetch_data_to_uri(f, f.name)
390 doc.packages.append(package)
391 doc.add_relationship(doc, "DESCRIBES", package)
392 # In the future, we might be able to do more fancy dependencies,
393 # but this should be sufficient for now
394 doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe)
395
396def get_license_list_version(license_data, d):
397 # Newer versions of the SPDX license list are SemVer ("MAJOR.MINOR.MICRO"),
398 # but SPDX 2 only uses "MAJOR.MINOR".
399 return ".".join(license_data["licenseListVersion"].split(".")[:2])
507 400
508 with spdx_deps_file.open("w") as f:
509 json.dump(deps, f)
510}
511# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
512addtask do_collect_spdx_deps after do_unpack
513do_collect_spdx_deps[depends] += "${PATCHDEPENDENCY}"
514do_collect_spdx_deps[deptask] = "do_create_spdx"
515do_collect_spdx_deps[dirs] = "${SPDXDIR}"
516 401
517python do_create_spdx() { 402python do_create_spdx() {
518 from datetime import datetime, timezone 403 from datetime import datetime, timezone
519 import oe.sbom 404 import oe.sbom
520 import oe.spdx 405 import oe.spdx
406 import oe.spdx_common
521 import uuid 407 import uuid
522 from pathlib import Path 408 from pathlib import Path
523 from contextlib import contextmanager 409 from contextlib import contextmanager
524 import oe.cve_check 410 import oe.cve_check
525 411
412 license_data = oe.spdx_common.load_spdx_license_data(d)
413
526 @contextmanager 414 @contextmanager
527 def optional_tarfile(name, guard, mode="w"): 415 def optional_tarfile(name, guard, mode="w"):
528 import tarfile 416 import tarfile
@@ -551,17 +439,17 @@ python do_create_spdx() {
551 doc = oe.spdx.SPDXDocument() 439 doc = oe.spdx.SPDXDocument()
552 440
553 doc.name = "recipe-" + d.getVar("PN") 441 doc.name = "recipe-" + d.getVar("PN")
554 doc.documentNamespace = get_doc_namespace(d, doc) 442 doc.documentNamespace = get_namespace(d, doc.name)
555 doc.creationInfo.created = creation_time 443 doc.creationInfo.created = creation_time
556 doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." 444 doc.creationInfo.comment = "This document was created by analyzing recipe files during the build."
557 doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] 445 doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d)
558 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 446 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
559 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 447 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
560 doc.creationInfo.creators.append("Person: N/A ()") 448 doc.creationInfo.creators.append("Person: N/A ()")
561 449
562 recipe = oe.spdx.SPDXPackage() 450 recipe = oe.spdx.SPDXPackage()
563 recipe.name = d.getVar("PN") 451 recipe.name = d.getVar("PN")
564 recipe.versionInfo = d.getVar("PV") 452 recipe.versionInfo = d.getVar("SPDX_PACKAGE_VERSION")
565 recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) 453 recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
566 recipe.supplier = d.getVar("SPDX_SUPPLIER") 454 recipe.supplier = d.getVar("SPDX_SUPPLIER")
567 if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): 455 if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d):
@@ -573,7 +461,7 @@ python do_create_spdx() {
573 461
574 license = d.getVar("LICENSE") 462 license = d.getVar("LICENSE")
575 if license: 463 if license:
576 recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) 464 recipe.licenseDeclared = convert_license_to_spdx(license, license_data, doc, d)
577 465
578 summary = d.getVar("SUMMARY") 466 summary = d.getVar("SUMMARY")
579 if summary: 467 if summary:
@@ -610,10 +498,10 @@ python do_create_spdx() {
610 498
611 add_download_packages(d, doc, recipe) 499 add_download_packages(d, doc, recipe)
612 500
613 if process_sources(d) and include_sources: 501 if oe.spdx_common.process_sources(d) and include_sources:
614 recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") 502 recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst")
615 with optional_tarfile(recipe_archive, archive_sources) as archive: 503 with optional_tarfile(recipe_archive, archive_sources) as archive:
616 spdx_get_src(d) 504 oe.spdx_common.get_patched_src(d)
617 505
618 add_package_files( 506 add_package_files(
619 d, 507 d,
@@ -655,10 +543,10 @@ python do_create_spdx() {
655 package_doc = oe.spdx.SPDXDocument() 543 package_doc = oe.spdx.SPDXDocument()
656 pkg_name = d.getVar("PKG:%s" % package) or package 544 pkg_name = d.getVar("PKG:%s" % package) or package
657 package_doc.name = pkg_name 545 package_doc.name = pkg_name
658 package_doc.documentNamespace = get_doc_namespace(d, package_doc) 546 package_doc.documentNamespace = get_namespace(d, package_doc.name)
659 package_doc.creationInfo.created = creation_time 547 package_doc.creationInfo.created = creation_time
660 package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." 548 package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build."
661 package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] 549 package_doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d)
662 package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 550 package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
663 package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 551 package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
664 package_doc.creationInfo.creators.append("Person: N/A ()") 552 package_doc.creationInfo.creators.append("Person: N/A ()")
@@ -670,8 +558,8 @@ python do_create_spdx() {
670 558
671 spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) 559 spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name)
672 spdx_package.name = pkg_name 560 spdx_package.name = pkg_name
673 spdx_package.versionInfo = d.getVar("PV") 561 spdx_package.versionInfo = d.getVar("SPDX_PACKAGE_VERSION")
674 spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) 562 spdx_package.licenseDeclared = convert_license_to_spdx(package_license, license_data, package_doc, d, found_licenses)
675 spdx_package.supplier = d.getVar("SPDX_SUPPLIER") 563 spdx_package.supplier = d.getVar("SPDX_SUPPLIER")
676 564
677 package_doc.packages.append(spdx_package) 565 package_doc.packages.append(spdx_package)
@@ -714,50 +602,16 @@ addtask do_create_spdx_setscene
714 602
715do_create_spdx[dirs] = "${SPDXWORK}" 603do_create_spdx[dirs] = "${SPDXWORK}"
716do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" 604do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}"
717do_create_spdx[depends] += "${PATCHDEPENDENCY}" 605do_create_spdx[depends] += " \
718 606 ${PATCHDEPENDENCY} \
719def collect_package_providers(d): 607 ${@create_spdx_source_deps(d)} \
720 from pathlib import Path 608"
721 import oe.sbom
722 import oe.spdx
723 import json
724
725 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
726
727 providers = {}
728
729 deps = collect_direct_deps(d, "do_create_spdx")
730 deps.append((d.getVar("PN"), d.getVar("BB_HASHFILENAME"), True))
731
732 for dep_pn, dep_hashfn, _ in deps:
733 localdata = d
734 recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
735 if not recipe_data:
736 localdata = bb.data.createCopy(d)
737 localdata.setVar("PKGDATA_DIR", "${PKGDATA_DIR_SDK}")
738 recipe_data = oe.packagedata.read_pkgdata(dep_pn, localdata)
739
740 for pkg in recipe_data.get("PACKAGES", "").split():
741
742 pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, localdata)
743 rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items())
744 rprovides.add(pkg)
745
746 if "PKG" in pkg_data:
747 pkg = pkg_data["PKG"]
748 rprovides.add(pkg)
749
750 for r in rprovides:
751 providers[r] = (pkg, dep_hashfn)
752
753 return providers
754
755collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA"
756 609
757python do_create_runtime_spdx() { 610python do_create_runtime_spdx() {
758 from datetime import datetime, timezone 611 from datetime import datetime, timezone
759 import oe.sbom 612 import oe.sbom
760 import oe.spdx 613 import oe.spdx
614 import oe.spdx_common
761 import oe.packagedata 615 import oe.packagedata
762 from pathlib import Path 616 from pathlib import Path
763 617
@@ -767,9 +621,11 @@ python do_create_runtime_spdx() {
767 621
768 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 622 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
769 623
770 providers = collect_package_providers(d) 624 license_data = oe.spdx_common.load_spdx_license_data(d)
625
626 providers = oe.spdx_common.collect_package_providers(d)
771 pkg_arch = d.getVar("SSTATE_PKGARCH") 627 pkg_arch = d.getVar("SSTATE_PKGARCH")
772 package_archs = d.getVar("SSTATE_ARCHS").split() 628 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split()
773 package_archs.reverse() 629 package_archs.reverse()
774 630
775 if not is_native: 631 if not is_native:
@@ -800,10 +656,10 @@ python do_create_runtime_spdx() {
800 656
801 runtime_doc = oe.spdx.SPDXDocument() 657 runtime_doc = oe.spdx.SPDXDocument()
802 runtime_doc.name = "runtime-" + pkg_name 658 runtime_doc.name = "runtime-" + pkg_name
803 runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) 659 runtime_doc.documentNamespace = get_namespace(localdata, runtime_doc.name)
804 runtime_doc.creationInfo.created = creation_time 660 runtime_doc.creationInfo.created = creation_time
805 runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." 661 runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies."
806 runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] 662 runtime_doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d)
807 runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 663 runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
808 runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 664 runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
809 runtime_doc.creationInfo.creators.append("Person: N/A ()") 665 runtime_doc.creationInfo.creators.append("Person: N/A ()")
@@ -875,7 +731,7 @@ python do_create_runtime_spdx() {
875 oe.sbom.write_doc(d, runtime_doc, pkg_arch, "runtime", spdx_deploy, indent=get_json_indent(d)) 731 oe.sbom.write_doc(d, runtime_doc, pkg_arch, "runtime", spdx_deploy, indent=get_json_indent(d))
876} 732}
877 733
878do_create_runtime_spdx[vardepsexclude] += "OVERRIDES SSTATE_ARCHS" 734do_create_runtime_spdx[vardepsexclude] += "OVERRIDES SPDX_MULTILIB_SSTATE_ARCHS"
879 735
880addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work 736addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work
881SSTATETASKS += "do_create_runtime_spdx" 737SSTATETASKS += "do_create_runtime_spdx"
@@ -891,60 +747,6 @@ do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}"
891do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" 747do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}"
892do_create_runtime_spdx[rdeptask] = "do_create_spdx" 748do_create_runtime_spdx[rdeptask] = "do_create_spdx"
893 749
894def spdx_get_src(d):
895 """
896 save patched source of the recipe in SPDX_WORKDIR.
897 """
898 import shutil
899 spdx_workdir = d.getVar('SPDXWORK')
900 spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE')
901 pn = d.getVar('PN')
902
903 workdir = d.getVar("WORKDIR")
904
905 try:
906 # The kernel class functions require it to be on work-shared, so we dont change WORKDIR
907 if not is_work_shared_spdx(d):
908 # Change the WORKDIR to make do_unpack do_patch run in another dir.
909 d.setVar('WORKDIR', spdx_workdir)
910 # Restore the original path to recipe's native sysroot (it's relative to WORKDIR).
911 d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)
912
913 # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the
914 # possibly requiring of the following tasks (such as some recipes's
915 # do_patch required 'B' existed).
916 bb.utils.mkdirhier(d.getVar('B'))
917
918 bb.build.exec_func('do_unpack', d)
919 # Copy source of kernel to spdx_workdir
920 if is_work_shared_spdx(d):
921 share_src = d.getVar('WORKDIR')
922 d.setVar('WORKDIR', spdx_workdir)
923 d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)
924 src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR')
925 bb.utils.mkdirhier(src_dir)
926 if bb.data.inherits_class('kernel',d):
927 share_src = d.getVar('STAGING_KERNEL_DIR')
928 cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/"
929 cmd_copy_shared_res = os.popen(cmd_copy_share).read()
930 bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res)
931
932 git_path = src_dir + "/.git"
933 if os.path.exists(git_path):
934 shutils.rmtree(git_path)
935
936 # Make sure gcc and kernel sources are patched only once
937 if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)):
938 bb.build.exec_func('do_patch', d)
939
940 # Some userland has no source.
941 if not os.path.exists( spdx_workdir ):
942 bb.utils.mkdirhier(spdx_workdir)
943 finally:
944 d.setVar("WORKDIR", workdir)
945
946spdx_get_src[vardepsexclude] += "STAGING_KERNEL_DIR"
947
948do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" 750do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
949do_rootfs[cleandirs] += "${SPDXIMAGEWORK}" 751do_rootfs[cleandirs] += "${SPDXIMAGEWORK}"
950 752
@@ -1002,6 +804,7 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
1002 import os 804 import os
1003 import oe.spdx 805 import oe.spdx
1004 import oe.sbom 806 import oe.sbom
807 import oe.spdx_common
1005 import io 808 import io
1006 import json 809 import json
1007 from datetime import timezone, datetime 810 from datetime import timezone, datetime
@@ -1009,8 +812,10 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
1009 import tarfile 812 import tarfile
1010 import bb.compress.zstd 813 import bb.compress.zstd
1011 814
1012 providers = collect_package_providers(d) 815 license_data = oe.spdx_common.load_spdx_license_data(d)
1013 package_archs = d.getVar("SSTATE_ARCHS").split() 816
817 providers = oe.spdx_common.collect_package_providers(d)
818 package_archs = d.getVar("SPDX_MULTILIB_SSTATE_ARCHS").split()
1014 package_archs.reverse() 819 package_archs.reverse()
1015 820
1016 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") 821 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -1019,68 +824,69 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
1019 824
1020 doc = oe.spdx.SPDXDocument() 825 doc = oe.spdx.SPDXDocument()
1021 doc.name = rootfs_name 826 doc.name = rootfs_name
1022 doc.documentNamespace = get_doc_namespace(d, doc) 827 doc.documentNamespace = get_namespace(d, doc.name)
1023 doc.creationInfo.created = creation_time 828 doc.creationInfo.created = creation_time
1024 doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." 829 doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build."
1025 doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] 830 doc.creationInfo.licenseListVersion = get_license_list_version(license_data, d)
1026 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") 831 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
1027 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) 832 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
1028 doc.creationInfo.creators.append("Person: N/A ()") 833 doc.creationInfo.creators.append("Person: N/A ()")
1029 834
1030 image = oe.spdx.SPDXPackage() 835 image = oe.spdx.SPDXPackage()
1031 image.name = d.getVar("PN") 836 image.name = d.getVar("PN")
1032 image.versionInfo = d.getVar("PV") 837 image.versionInfo = d.getVar("SPDX_PACKAGE_VERSION")
1033 image.SPDXID = rootfs_spdxid 838 image.SPDXID = rootfs_spdxid
1034 image.supplier = d.getVar("SPDX_SUPPLIER") 839 image.supplier = d.getVar("SPDX_SUPPLIER")
1035 840
1036 doc.packages.append(image) 841 doc.packages.append(image)
1037 842
1038 for name in sorted(packages.keys()): 843 if packages:
1039 if name not in providers: 844 for name in sorted(packages.keys()):
1040 bb.fatal("Unable to find SPDX provider for '%s'" % name) 845 if name not in providers:
1041 846 bb.fatal("Unable to find SPDX provider for '%s'" % name)
1042 pkg_name, pkg_hashfn = providers[name]
1043 847
1044 pkg_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, pkg_name, pkg_hashfn) 848 pkg_name, pkg_hashfn = providers[name]
1045 if not pkg_spdx_path:
1046 bb.fatal("No SPDX file found for package %s, %s" % (pkg_name, pkg_hashfn))
1047 849
1048 pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) 850 pkg_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, pkg_name, pkg_hashfn)
851 if not pkg_spdx_path:
852 bb.fatal("No SPDX file found for package %s, %s" % (pkg_name, pkg_hashfn))
1049 853
1050 for p in pkg_doc.packages: 854 pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path)
1051 if p.name == name:
1052 pkg_ref = oe.spdx.SPDXExternalDocumentRef()
1053 pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name
1054 pkg_ref.spdxDocument = pkg_doc.documentNamespace
1055 pkg_ref.checksum.algorithm = "SHA1"
1056 pkg_ref.checksum.checksumValue = pkg_doc_sha1
1057 855
1058 doc.externalDocumentRefs.append(pkg_ref) 856 for p in pkg_doc.packages:
1059 doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) 857 if p.name == name:
1060 break 858 pkg_ref = oe.spdx.SPDXExternalDocumentRef()
1061 else: 859 pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name
1062 bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) 860 pkg_ref.spdxDocument = pkg_doc.documentNamespace
861 pkg_ref.checksum.algorithm = "SHA1"
862 pkg_ref.checksum.checksumValue = pkg_doc_sha1
1063 863
1064 runtime_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "runtime-" + name, pkg_hashfn) 864 doc.externalDocumentRefs.append(pkg_ref)
1065 if not runtime_spdx_path: 865 doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID))
1066 bb.fatal("No runtime SPDX document found for %s, %s" % (name, pkg_hashfn)) 866 break
1067 867 else:
1068 runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) 868 bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path))
1069 869
1070 runtime_ref = oe.spdx.SPDXExternalDocumentRef() 870 runtime_spdx_path = oe.sbom.doc_find_by_hashfn(deploy_dir_spdx, package_archs, "runtime-" + name, pkg_hashfn)
1071 runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name 871 if not runtime_spdx_path:
1072 runtime_ref.spdxDocument = runtime_doc.documentNamespace 872 bb.fatal("No runtime SPDX document found for %s, %s" % (name, pkg_hashfn))
1073 runtime_ref.checksum.algorithm = "SHA1" 873
1074 runtime_ref.checksum.checksumValue = runtime_doc_sha1 874 runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path)
1075 875
1076 # "OTHER" isn't ideal here, but I can't find a relationship that makes sense 876 runtime_ref = oe.spdx.SPDXExternalDocumentRef()
1077 doc.externalDocumentRefs.append(runtime_ref) 877 runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name
1078 doc.add_relationship( 878 runtime_ref.spdxDocument = runtime_doc.documentNamespace
1079 image, 879 runtime_ref.checksum.algorithm = "SHA1"
1080 "OTHER", 880 runtime_ref.checksum.checksumValue = runtime_doc_sha1
1081 "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), 881
1082 comment="Runtime dependencies for %s" % name 882 # "OTHER" isn't ideal here, but I can't find a relationship that makes sense
1083 ) 883 doc.externalDocumentRefs.append(runtime_ref)
884 doc.add_relationship(
885 image,
886 "OTHER",
887 "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID),
888 comment="Runtime dependencies for %s" % name
889 )
1084 bb.utils.mkdirhier(spdx_workdir) 890 bb.utils.mkdirhier(spdx_workdir)
1085 image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json") 891 image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json")
1086 892
@@ -1161,4 +967,4 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx
1161 967
1162 tar.addfile(info, fileobj=index_str) 968 tar.addfile(info, fileobj=index_str)
1163 969
1164combine_spdx[vardepsexclude] += "BB_NUMBER_THREADS SSTATE_ARCHS" 970combine_spdx[vardepsexclude] += "BB_NUMBER_THREADS SPDX_MULTILIB_SSTATE_ARCHS"