summaryrefslogtreecommitdiffstats
path: root/meta/classes
diff options
context:
space:
mode:
authorJoshua Watt <JPEWhacker@gmail.com>2022-11-08 14:07:02 -0600
committerRichard Purdie <richard.purdie@linuxfoundation.org>2022-11-11 13:43:40 +0000
commit02691744256d4f267f37190e30b03b3bc3e477fc (patch)
tree2ba6eb23db297915da63f5e47d4d39ee0b39269e /meta/classes
parent25e2663a878c7cd4463652757f7c36ab5ace9b39 (diff)
downloadpoky-02691744256d4f267f37190e30b03b3bc3e477fc.tar.gz
classes: create-spdx: Move to version specific class
In expectation of SPDX 3.0 support, move the create-spdx.bbclass -> create-spdx-2.2.bbclass. The create-spdx.bbclass class still exists and can be used if a user doesn't care about which specific version of SPDX they get. (From OE-Core rev: 9ec01fe3e59be66331c14ab5391ecb0b6f140c22) Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> Signed-off-by: Alexandre Belloni <alexandre.belloni@bootlin.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/classes')
-rw-r--r--meta/classes/create-spdx-2.2.bbclass1025
-rw-r--r--meta/classes/create-spdx.bbclass1023
2 files changed, 1028 insertions, 1020 deletions
diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass
new file mode 100644
index 0000000000..af6afcc653
--- /dev/null
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -0,0 +1,1025 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${MACHINE}"
8
9# The product name that the CVE database uses. Defaults to BPN, but may need to
10# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff).
11CVE_PRODUCT ??= "${BPN}"
12CVE_VERSION ??= "${PV}"
13
14SPDXDIR ??= "${WORKDIR}/spdx"
15SPDXDEPLOY = "${SPDXDIR}/deploy"
16SPDXWORK = "${SPDXDIR}/work"
17
18SPDX_TOOL_NAME ??= "oe-spdx-creator"
19SPDX_TOOL_VERSION ??= "1.0"
20
21SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy"
22
23SPDX_INCLUDE_SOURCES ??= "0"
24SPDX_ARCHIVE_SOURCES ??= "0"
25SPDX_ARCHIVE_PACKAGED ??= "0"
26
27SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org"
28SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc"
29SPDX_PRETTY ??= "0"
30
31SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
32
33SPDX_ORG ??= "OpenEmbedded ()"
34SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
35SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \
36 this recipe. For SPDX documents create using this class during the build, this \
37 is the contact information for the person or organization who is doing the \
38 build."
39
40def extract_licenses(filename):
41 import re
42
43 lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
44
45 try:
46 with open(filename, 'rb') as f:
47 size = min(15000, os.stat(filename).st_size)
48 txt = f.read(size)
49 licenses = re.findall(lic_regex, txt)
50 if licenses:
51 ascii_licenses = [lic.decode('ascii') for lic in licenses]
52 return ascii_licenses
53 except Exception as e:
54 bb.warn(f"Exception reading {filename}: {e}")
55 return None
56
57def get_doc_namespace(d, doc):
58 import uuid
59 namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
60 return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name)))
61
62def create_annotation(d, comment):
63 from datetime import datetime, timezone
64
65 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
66 annotation = oe.spdx.SPDXAnnotation()
67 annotation.annotationDate = creation_time
68 annotation.annotationType = "OTHER"
69 annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION"))
70 annotation.comment = comment
71 return annotation
72
73def recipe_spdx_is_native(d, recipe):
74 return any(a.annotationType == "OTHER" and
75 a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and
76 a.comment == "isNative" for a in recipe.annotations)
77
78def is_work_shared_spdx(d):
79 return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR'))
80
81def get_json_indent(d):
82 if d.getVar("SPDX_PRETTY") == "1":
83 return 2
84 return None
85
86python() {
87 import json
88 if d.getVar("SPDX_LICENSE_DATA"):
89 return
90
91 with open(d.getVar("SPDX_LICENSES"), "r") as f:
92 data = json.load(f)
93 # Transform the license array to a dictionary
94 data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
95 d.setVar("SPDX_LICENSE_DATA", data)
96}
97
98def convert_license_to_spdx(lic, document, d, existing={}):
99 from pathlib import Path
100 import oe.spdx
101
102 license_data = d.getVar("SPDX_LICENSE_DATA")
103 extracted = {}
104
105 def add_extracted_license(ident, name):
106 nonlocal document
107
108 if name in extracted:
109 return
110
111 extracted_info = oe.spdx.SPDXExtractedLicensingInfo()
112 extracted_info.name = name
113 extracted_info.licenseId = ident
114 extracted_info.extractedText = None
115
116 if name == "PD":
117 # Special-case this.
118 extracted_info.extractedText = "Software released to the public domain"
119 else:
120 # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH
121 for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split():
122 try:
123 with (Path(directory) / name).open(errors="replace") as f:
124 extracted_info.extractedText = f.read()
125 break
126 except FileNotFoundError:
127 pass
128 if extracted_info.extractedText is None:
129 # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set
130 filename = d.getVarFlag('NO_GENERIC_LICENSE', name)
131 if filename:
132 filename = d.expand("${S}/" + filename)
133 with open(filename, errors="replace") as f:
134 extracted_info.extractedText = f.read()
135 else:
136 bb.error("Cannot find any text for license %s" % name)
137
138 extracted[name] = extracted_info
139 document.hasExtractedLicensingInfos.append(extracted_info)
140
141 def convert(l):
142 if l == "(" or l == ")":
143 return l
144
145 if l == "&":
146 return "AND"
147
148 if l == "|":
149 return "OR"
150
151 if l == "CLOSED":
152 return "NONE"
153
154 spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l
155 if spdx_license in license_data["licenses"]:
156 return spdx_license
157
158 try:
159 spdx_license = existing[l]
160 except KeyError:
161 spdx_license = "LicenseRef-" + l
162 add_extracted_license(spdx_license, l)
163
164 return spdx_license
165
166 lic_split = lic.replace("(", " ( ").replace(")", " ) ").split()
167
168 return ' '.join(convert(l) for l in lic_split)
169
170def process_sources(d):
171 pn = d.getVar('PN')
172 assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
173 if pn in assume_provided:
174 for p in d.getVar("PROVIDES").split():
175 if p != pn:
176 pn = p
177 break
178
179 # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
180 # so avoid archiving source here.
181 if pn.startswith('glibc-locale'):
182 return False
183 if d.getVar('PN') == "libtool-cross":
184 return False
185 if d.getVar('PN') == "libgcc-initial":
186 return False
187 if d.getVar('PN') == "shadow-sysroot":
188 return False
189
190 # We just archive gcc-source for all the gcc related recipes
191 if d.getVar('BPN') in ['gcc', 'libgcc']:
192 bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
193 return False
194
195 return True
196
197
198def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]):
199 from pathlib import Path
200 import oe.spdx
201 import hashlib
202
203 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
204 if source_date_epoch:
205 source_date_epoch = int(source_date_epoch)
206
207 sha1s = []
208 spdx_files = []
209
210 file_counter = 1
211 for subdir, dirs, files in os.walk(topdir):
212 dirs[:] = [d for d in dirs if d not in ignore_dirs]
213 if subdir == str(topdir):
214 dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs]
215
216 for file in files:
217 filepath = Path(subdir) / file
218 filename = str(filepath.relative_to(topdir))
219
220 if not filepath.is_symlink() and filepath.is_file():
221 spdx_file = oe.spdx.SPDXFile()
222 spdx_file.SPDXID = get_spdxid(file_counter)
223 for t in get_types(filepath):
224 spdx_file.fileTypes.append(t)
225 spdx_file.fileName = filename
226
227 if archive is not None:
228 with filepath.open("rb") as f:
229 info = archive.gettarinfo(fileobj=f)
230 info.name = filename
231 info.uid = 0
232 info.gid = 0
233 info.uname = "root"
234 info.gname = "root"
235
236 if source_date_epoch is not None and info.mtime > source_date_epoch:
237 info.mtime = source_date_epoch
238
239 archive.addfile(info, f)
240
241 sha1 = bb.utils.sha1_file(filepath)
242 sha1s.append(sha1)
243 spdx_file.checksums.append(oe.spdx.SPDXChecksum(
244 algorithm="SHA1",
245 checksumValue=sha1,
246 ))
247 spdx_file.checksums.append(oe.spdx.SPDXChecksum(
248 algorithm="SHA256",
249 checksumValue=bb.utils.sha256_file(filepath),
250 ))
251
252 if "SOURCE" in spdx_file.fileTypes:
253 extracted_lics = extract_licenses(filepath)
254 if extracted_lics:
255 spdx_file.licenseInfoInFiles = extracted_lics
256
257 doc.files.append(spdx_file)
258 doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file)
259 spdx_pkg.hasFiles.append(spdx_file.SPDXID)
260
261 spdx_files.append(spdx_file)
262
263 file_counter += 1
264
265 sha1s.sort()
266 verifier = hashlib.sha1()
267 for v in sha1s:
268 verifier.update(v.encode("utf-8"))
269 spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest()
270
271 return spdx_files
272
273
274def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources):
275 from pathlib import Path
276 import hashlib
277 import oe.packagedata
278 import oe.spdx
279
280 debug_search_paths = [
281 Path(d.getVar('PKGD')),
282 Path(d.getVar('STAGING_DIR_TARGET')),
283 Path(d.getVar('STAGING_DIR_NATIVE')),
284 Path(d.getVar('STAGING_KERNEL_DIR')),
285 ]
286
287 pkg_data = oe.packagedata.read_subpkgdata_extended(package, d)
288
289 if pkg_data is None:
290 return
291
292 for file_path, file_data in pkg_data["files_info"].items():
293 if not "debugsrc" in file_data:
294 continue
295
296 for pkg_file in package_files:
297 if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"):
298 break
299 else:
300 bb.fatal("No package file found for %s" % str(file_path))
301 continue
302
303 for debugsrc in file_data["debugsrc"]:
304 ref_id = "NOASSERTION"
305 for search in debug_search_paths:
306 if debugsrc.startswith("/usr/src/kernel"):
307 debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '')
308 else:
309 debugsrc_path = search / debugsrc.lstrip("/")
310 if not debugsrc_path.exists():
311 continue
312
313 file_sha256 = bb.utils.sha256_file(debugsrc_path)
314
315 if file_sha256 in sources:
316 source_file = sources[file_sha256]
317
318 doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace)
319 if doc_ref is None:
320 doc_ref = oe.spdx.SPDXExternalDocumentRef()
321 doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name
322 doc_ref.spdxDocument = source_file.doc.documentNamespace
323 doc_ref.checksum.algorithm = "SHA1"
324 doc_ref.checksum.checksumValue = source_file.doc_sha1
325 package_doc.externalDocumentRefs.append(doc_ref)
326
327 ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID)
328 else:
329 bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256))
330 break
331 else:
332 bb.debug(1, "Debug source %s not found" % debugsrc)
333
334 package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc)
335
336def collect_dep_recipes(d, doc, spdx_recipe):
337 from pathlib import Path
338 import oe.sbom
339 import oe.spdx
340
341 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
342
343 dep_recipes = []
344 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
345 deps = sorted(set(
346 dep[0] for dep in taskdepdata.values() if
347 dep[1] == "do_create_spdx" and dep[0] != d.getVar("PN")
348 ))
349 for dep_pn in deps:
350 dep_recipe_path = deploy_dir_spdx / "recipes" / ("recipe-%s.spdx.json" % dep_pn)
351
352 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path)
353
354 for pkg in spdx_dep_doc.packages:
355 if pkg.name == dep_pn:
356 spdx_dep_recipe = pkg
357 break
358 else:
359 continue
360
361 dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe))
362
363 dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef()
364 dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name
365 dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace
366 dep_recipe_ref.checksum.algorithm = "SHA1"
367 dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1
368
369 doc.externalDocumentRefs.append(dep_recipe_ref)
370
371 doc.add_relationship(
372 "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID),
373 "BUILD_DEPENDENCY_OF",
374 spdx_recipe
375 )
376
377 return dep_recipes
378
379collect_dep_recipes[vardepsexclude] += "BB_TASKDEPDATA"
380
381
382def collect_dep_sources(d, dep_recipes):
383 import oe.sbom
384
385 sources = {}
386 for dep in dep_recipes:
387 # Don't collect sources from native recipes as they
388 # match non-native sources also.
389 if recipe_spdx_is_native(d, dep.recipe):
390 continue
391 recipe_files = set(dep.recipe.hasFiles)
392
393 for spdx_file in dep.doc.files:
394 if spdx_file.SPDXID not in recipe_files:
395 continue
396
397 if "SOURCE" in spdx_file.fileTypes:
398 for checksum in spdx_file.checksums:
399 if checksum.algorithm == "SHA256":
400 sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file)
401 break
402
403 return sources
404
405
406python do_create_spdx() {
407 from datetime import datetime, timezone
408 import oe.sbom
409 import oe.spdx
410 import uuid
411 from pathlib import Path
412 from contextlib import contextmanager
413 import oe.cve_check
414
415 @contextmanager
416 def optional_tarfile(name, guard, mode="w"):
417 import tarfile
418 import bb.compress.zstd
419
420 num_threads = int(d.getVar("BB_NUMBER_THREADS"))
421
422 if guard:
423 name.parent.mkdir(parents=True, exist_ok=True)
424 with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f:
425 with tarfile.open(fileobj=f, mode=mode + "|") as tf:
426 yield tf
427 else:
428 yield None
429
430
431 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
432 spdx_workdir = Path(d.getVar("SPDXWORK"))
433 include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
434 archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1"
435 archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1"
436
437 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
438
439 doc = oe.spdx.SPDXDocument()
440
441 doc.name = "recipe-" + d.getVar("PN")
442 doc.documentNamespace = get_doc_namespace(d, doc)
443 doc.creationInfo.created = creation_time
444 doc.creationInfo.comment = "This document was created by analyzing recipe files during the build."
445 doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
446 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
447 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
448 doc.creationInfo.creators.append("Person: N/A ()")
449
450 recipe = oe.spdx.SPDXPackage()
451 recipe.name = d.getVar("PN")
452 recipe.versionInfo = d.getVar("PV")
453 recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
454 recipe.supplier = d.getVar("SPDX_SUPPLIER")
455 if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d):
456 recipe.annotations.append(create_annotation(d, "isNative"))
457
458 for s in d.getVar('SRC_URI').split():
459 if not s.startswith("file://"):
460 s = s.split(';')[0]
461 recipe.downloadLocation = s
462 break
463 else:
464 recipe.downloadLocation = "NOASSERTION"
465
466 homepage = d.getVar("HOMEPAGE")
467 if homepage:
468 recipe.homepage = homepage
469
470 license = d.getVar("LICENSE")
471 if license:
472 recipe.licenseDeclared = convert_license_to_spdx(license, doc, d)
473
474 summary = d.getVar("SUMMARY")
475 if summary:
476 recipe.summary = summary
477
478 description = d.getVar("DESCRIPTION")
479 if description:
480 recipe.description = description
481
482 # Some CVEs may be patched during the build process without incrementing the version number,
483 # so querying for CVEs based on the CPE id can lead to false positives. To account for this,
484 # save the CVEs fixed by patches to source information field in the SPDX.
485 patched_cves = oe.cve_check.get_patched_cves(d)
486 patched_cves = list(patched_cves)
487 patched_cves = ' '.join(patched_cves)
488 if patched_cves:
489 recipe.sourceInfo = "CVEs fixed: " + patched_cves
490
491 cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION"))
492 if cpe_ids:
493 for cpe_id in cpe_ids:
494 cpe = oe.spdx.SPDXExternalReference()
495 cpe.referenceCategory = "SECURITY"
496 cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type"
497 cpe.referenceLocator = cpe_id
498 recipe.externalRefs.append(cpe)
499
500 doc.packages.append(recipe)
501 doc.add_relationship(doc, "DESCRIBES", recipe)
502
503 if process_sources(d) and include_sources:
504 recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst")
505 with optional_tarfile(recipe_archive, archive_sources) as archive:
506 spdx_get_src(d)
507
508 add_package_files(
509 d,
510 doc,
511 recipe,
512 spdx_workdir,
513 lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter),
514 lambda filepath: ["SOURCE"],
515 ignore_dirs=[".git"],
516 ignore_top_level_dirs=["temp"],
517 archive=archive,
518 )
519
520 if archive is not None:
521 recipe.packageFileName = str(recipe_archive.name)
522
523 dep_recipes = collect_dep_recipes(d, doc, recipe)
524
525 doc_sha1 = oe.sbom.write_doc(d, doc, "recipes", indent=get_json_indent(d))
526 dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe))
527
528 recipe_ref = oe.spdx.SPDXExternalDocumentRef()
529 recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name
530 recipe_ref.spdxDocument = doc.documentNamespace
531 recipe_ref.checksum.algorithm = "SHA1"
532 recipe_ref.checksum.checksumValue = doc_sha1
533
534 sources = collect_dep_sources(d, dep_recipes)
535 found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos}
536
537 if not recipe_spdx_is_native(d, recipe):
538 bb.build.exec_func("read_subpackage_metadata", d)
539
540 pkgdest = Path(d.getVar("PKGDEST"))
541 for package in d.getVar("PACKAGES").split():
542 if not oe.packagedata.packaged(package, d):
543 continue
544
545 package_doc = oe.spdx.SPDXDocument()
546 pkg_name = d.getVar("PKG:%s" % package) or package
547 package_doc.name = pkg_name
548 package_doc.documentNamespace = get_doc_namespace(d, package_doc)
549 package_doc.creationInfo.created = creation_time
550 package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build."
551 package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
552 package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
553 package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
554 package_doc.creationInfo.creators.append("Person: N/A ()")
555 package_doc.externalDocumentRefs.append(recipe_ref)
556
557 package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE")
558
559 spdx_package = oe.spdx.SPDXPackage()
560
561 spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name)
562 spdx_package.name = pkg_name
563 spdx_package.versionInfo = d.getVar("PV")
564 spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses)
565 spdx_package.supplier = d.getVar("SPDX_SUPPLIER")
566
567 package_doc.packages.append(spdx_package)
568
569 package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID))
570 package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package)
571
572 package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst")
573 with optional_tarfile(package_archive, archive_packaged) as archive:
574 package_files = add_package_files(
575 d,
576 package_doc,
577 spdx_package,
578 pkgdest / package,
579 lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter),
580 lambda filepath: ["BINARY"],
581 ignore_top_level_dirs=['CONTROL', 'DEBIAN'],
582 archive=archive,
583 )
584
585 if archive is not None:
586 spdx_package.packageFileName = str(package_archive.name)
587
588 add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources)
589
590 oe.sbom.write_doc(d, package_doc, "packages", indent=get_json_indent(d))
591}
592# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
593addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work
594
595SSTATETASKS += "do_create_spdx"
596do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
597do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
598
599python do_create_spdx_setscene () {
600 sstate_setscene(d)
601}
602addtask do_create_spdx_setscene
603
604do_create_spdx[dirs] = "${SPDXWORK}"
605do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}"
606do_create_spdx[depends] += "${PATCHDEPENDENCY}"
607do_create_spdx[deptask] = "do_create_spdx"
608
609def collect_package_providers(d):
610 from pathlib import Path
611 import oe.sbom
612 import oe.spdx
613 import json
614
615 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
616
617 providers = {}
618
619 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
620 deps = sorted(set(
621 dep[0] for dep in taskdepdata.values() if dep[0] != d.getVar("PN")
622 ))
623 deps.append(d.getVar("PN"))
624
625 for dep_pn in deps:
626 recipe_data = oe.packagedata.read_pkgdata(dep_pn, d)
627
628 for pkg in recipe_data.get("PACKAGES", "").split():
629
630 pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, d)
631 rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items())
632 rprovides.add(pkg)
633
634 for r in rprovides:
635 providers[r] = pkg
636
637 return providers
638
639collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA"
640
641python do_create_runtime_spdx() {
642 from datetime import datetime, timezone
643 import oe.sbom
644 import oe.spdx
645 import oe.packagedata
646 from pathlib import Path
647
648 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
649 spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY"))
650 is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d)
651
652 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
653
654 providers = collect_package_providers(d)
655
656 if not is_native:
657 bb.build.exec_func("read_subpackage_metadata", d)
658
659 dep_package_cache = {}
660
661 pkgdest = Path(d.getVar("PKGDEST"))
662 for package in d.getVar("PACKAGES").split():
663 localdata = bb.data.createCopy(d)
664 pkg_name = d.getVar("PKG:%s" % package) or package
665 localdata.setVar("PKG", pkg_name)
666 localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package)
667
668 if not oe.packagedata.packaged(package, localdata):
669 continue
670
671 pkg_spdx_path = deploy_dir_spdx / "packages" / (pkg_name + ".spdx.json")
672
673 package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path)
674
675 for p in package_doc.packages:
676 if p.name == pkg_name:
677 spdx_package = p
678 break
679 else:
680 bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path))
681
682 runtime_doc = oe.spdx.SPDXDocument()
683 runtime_doc.name = "runtime-" + pkg_name
684 runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc)
685 runtime_doc.creationInfo.created = creation_time
686 runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies."
687 runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
688 runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
689 runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
690 runtime_doc.creationInfo.creators.append("Person: N/A ()")
691
692 package_ref = oe.spdx.SPDXExternalDocumentRef()
693 package_ref.externalDocumentId = "DocumentRef-package-" + package
694 package_ref.spdxDocument = package_doc.documentNamespace
695 package_ref.checksum.algorithm = "SHA1"
696 package_ref.checksum.checksumValue = package_doc_sha1
697
698 runtime_doc.externalDocumentRefs.append(package_ref)
699
700 runtime_doc.add_relationship(
701 runtime_doc.SPDXID,
702 "AMENDS",
703 "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID)
704 )
705
706 deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "")
707 seen_deps = set()
708 for dep, _ in deps.items():
709 if dep in seen_deps:
710 continue
711
712 if dep not in providers:
713 continue
714
715 dep = providers[dep]
716
717 if not oe.packagedata.packaged(dep, localdata):
718 continue
719
720 dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d)
721 dep_pkg = dep_pkg_data["PKG"]
722
723 if dep in dep_package_cache:
724 (dep_spdx_package, dep_package_ref) = dep_package_cache[dep]
725 else:
726 dep_path = deploy_dir_spdx / "packages" / ("%s.spdx.json" % dep_pkg)
727
728 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path)
729
730 for pkg in spdx_dep_doc.packages:
731 if pkg.name == dep_pkg:
732 dep_spdx_package = pkg
733 break
734 else:
735 bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path))
736
737 dep_package_ref = oe.spdx.SPDXExternalDocumentRef()
738 dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name
739 dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace
740 dep_package_ref.checksum.algorithm = "SHA1"
741 dep_package_ref.checksum.checksumValue = spdx_dep_sha1
742
743 dep_package_cache[dep] = (dep_spdx_package, dep_package_ref)
744
745 runtime_doc.externalDocumentRefs.append(dep_package_ref)
746
747 runtime_doc.add_relationship(
748 "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID),
749 "RUNTIME_DEPENDENCY_OF",
750 "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID)
751 )
752 seen_deps.add(dep)
753
754 oe.sbom.write_doc(d, runtime_doc, "runtime", spdx_deploy, indent=get_json_indent(d))
755}
756
757addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work
758SSTATETASKS += "do_create_runtime_spdx"
759do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}"
760do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
761
762python do_create_runtime_spdx_setscene () {
763 sstate_setscene(d)
764}
765addtask do_create_runtime_spdx_setscene
766
767do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}"
768do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}"
769do_create_runtime_spdx[rdeptask] = "do_create_spdx"
770
771def spdx_get_src(d):
772 """
773 save patched source of the recipe in SPDX_WORKDIR.
774 """
775 import shutil
776 spdx_workdir = d.getVar('SPDXWORK')
777 spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE')
778 pn = d.getVar('PN')
779
780 workdir = d.getVar("WORKDIR")
781
782 try:
783 # The kernel class functions require it to be on work-shared, so we dont change WORKDIR
784 if not is_work_shared_spdx(d):
785 # Change the WORKDIR to make do_unpack do_patch run in another dir.
786 d.setVar('WORKDIR', spdx_workdir)
787 # Restore the original path to recipe's native sysroot (it's relative to WORKDIR).
788 d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)
789
790 # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the
791 # possibly requiring of the following tasks (such as some recipes's
792 # do_patch required 'B' existed).
793 bb.utils.mkdirhier(d.getVar('B'))
794
795 bb.build.exec_func('do_unpack', d)
796 # Copy source of kernel to spdx_workdir
797 if is_work_shared_spdx(d):
798 d.setVar('WORKDIR', spdx_workdir)
799 d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)
800 src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR')
801 bb.utils.mkdirhier(src_dir)
802 if bb.data.inherits_class('kernel',d):
803 share_src = d.getVar('STAGING_KERNEL_DIR')
804 cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/"
805 cmd_copy_kernel_result = os.popen(cmd_copy_share).read()
806 bb.note("cmd_copy_kernel_result = " + cmd_copy_kernel_result)
807
808 git_path = src_dir + "/.git"
809 if os.path.exists(git_path):
810 shutils.rmtree(git_path)
811
812 # Make sure gcc and kernel sources are patched only once
813 if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)):
814 bb.build.exec_func('do_patch', d)
815
816 # Some userland has no source.
817 if not os.path.exists( spdx_workdir ):
818 bb.utils.mkdirhier(spdx_workdir)
819 finally:
820 d.setVar("WORKDIR", workdir)
821
822do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
823
824ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; "
825
826do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
827POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx; "
828POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx; "
829
830python image_combine_spdx() {
831 import os
832 import oe.sbom
833 from pathlib import Path
834 from oe.rootfs import image_list_installed_packages
835
836 image_name = d.getVar("IMAGE_NAME")
837 image_link_name = d.getVar("IMAGE_LINK_NAME")
838 imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
839 img_spdxid = oe.sbom.get_image_spdxid(image_name)
840 packages = image_list_installed_packages(d)
841
842 combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages)
843
844 def make_image_link(target_path, suffix):
845 if image_link_name:
846 link = imgdeploydir / (image_link_name + suffix)
847 if link != target_path:
848 link.symlink_to(os.path.relpath(target_path, link.parent))
849
850 image_spdx_path = imgdeploydir / (image_name + ".spdx.json")
851 make_image_link(image_spdx_path, ".spdx.json")
852 spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst")
853 make_image_link(spdx_tar_path, ".spdx.tar.zst")
854 spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json")
855 make_image_link(spdx_index_path, ".spdx.index.json")
856}
857
858python sdk_host_combine_spdx() {
859 sdk_combine_spdx(d, "host")
860}
861
862python sdk_target_combine_spdx() {
863 sdk_combine_spdx(d, "target")
864}
865
866def sdk_combine_spdx(d, sdk_type):
867 import oe.sbom
868 from pathlib import Path
869 from oe.sdk import sdk_list_installed_packages
870
871 sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type
872 sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR"))
873 sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name)
874 sdk_packages = sdk_list_installed_packages(d, sdk_type == "target")
875 combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages)
876
877def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages):
878 import os
879 import oe.spdx
880 import oe.sbom
881 import io
882 import json
883 from datetime import timezone, datetime
884 from pathlib import Path
885 import tarfile
886 import bb.compress.zstd
887
888 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
889 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
890 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
891
892 doc = oe.spdx.SPDXDocument()
893 doc.name = rootfs_name
894 doc.documentNamespace = get_doc_namespace(d, doc)
895 doc.creationInfo.created = creation_time
896 doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build."
897 doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
898 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
899 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
900 doc.creationInfo.creators.append("Person: N/A ()")
901
902 image = oe.spdx.SPDXPackage()
903 image.name = d.getVar("PN")
904 image.versionInfo = d.getVar("PV")
905 image.SPDXID = rootfs_spdxid
906 image.supplier = d.getVar("SPDX_SUPPLIER")
907
908 doc.packages.append(image)
909
910 for name in sorted(packages.keys()):
911 pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json")
912 pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path)
913
914 for p in pkg_doc.packages:
915 if p.name == name:
916 pkg_ref = oe.spdx.SPDXExternalDocumentRef()
917 pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name
918 pkg_ref.spdxDocument = pkg_doc.documentNamespace
919 pkg_ref.checksum.algorithm = "SHA1"
920 pkg_ref.checksum.checksumValue = pkg_doc_sha1
921
922 doc.externalDocumentRefs.append(pkg_ref)
923 doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID))
924 break
925 else:
926 bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path))
927
928 runtime_spdx_path = deploy_dir_spdx / "runtime" / ("runtime-" + name + ".spdx.json")
929 runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path)
930
931 runtime_ref = oe.spdx.SPDXExternalDocumentRef()
932 runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name
933 runtime_ref.spdxDocument = runtime_doc.documentNamespace
934 runtime_ref.checksum.algorithm = "SHA1"
935 runtime_ref.checksum.checksumValue = runtime_doc_sha1
936
937 # "OTHER" isn't ideal here, but I can't find a relationship that makes sense
938 doc.externalDocumentRefs.append(runtime_ref)
939 doc.add_relationship(
940 image,
941 "OTHER",
942 "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID),
943 comment="Runtime dependencies for %s" % name
944 )
945
946 image_spdx_path = rootfs_deploydir / (rootfs_name + ".spdx.json")
947
948 with image_spdx_path.open("wb") as f:
949 doc.to_json(f, sort_keys=True, indent=get_json_indent(d))
950
951 num_threads = int(d.getVar("BB_NUMBER_THREADS"))
952
953 visited_docs = set()
954
955 index = {"documents": []}
956
957 spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst")
958 with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f:
959 with tarfile.open(fileobj=f, mode="w|") as tar:
960 def collect_spdx_document(path):
961 nonlocal tar
962 nonlocal deploy_dir_spdx
963 nonlocal source_date_epoch
964 nonlocal index
965
966 if path in visited_docs:
967 return
968
969 visited_docs.add(path)
970
971 with path.open("rb") as f:
972 doc, sha1 = oe.sbom.read_doc(f)
973 f.seek(0)
974
975 if doc.documentNamespace in visited_docs:
976 return
977
978 bb.note("Adding SPDX document %s" % path)
979 visited_docs.add(doc.documentNamespace)
980 info = tar.gettarinfo(fileobj=f)
981
982 info.name = doc.name + ".spdx.json"
983 info.uid = 0
984 info.gid = 0
985 info.uname = "root"
986 info.gname = "root"
987
988 if source_date_epoch is not None and info.mtime > int(source_date_epoch):
989 info.mtime = int(source_date_epoch)
990
991 tar.addfile(info, f)
992
993 index["documents"].append({
994 "filename": info.name,
995 "documentNamespace": doc.documentNamespace,
996 "sha1": sha1,
997 })
998
999 for ref in doc.externalDocumentRefs:
1000 ref_path = deploy_dir_spdx / "by-namespace" / ref.spdxDocument.replace("/", "_")
1001 collect_spdx_document(ref_path)
1002
1003 collect_spdx_document(image_spdx_path)
1004
1005 index["documents"].sort(key=lambda x: x["filename"])
1006
1007 index_str = io.BytesIO(json.dumps(
1008 index,
1009 sort_keys=True,
1010 indent=get_json_indent(d),
1011 ).encode("utf-8"))
1012
1013 info = tarfile.TarInfo()
1014 info.name = "index.json"
1015 info.size = len(index_str.getvalue())
1016 info.uid = 0
1017 info.gid = 0
1018 info.uname = "root"
1019 info.gname = "root"
1020
1021 tar.addfile(info, fileobj=index_str)
1022
1023 spdx_index_path = rootfs_deploydir / (rootfs_name + ".spdx.index.json")
1024 with spdx_index_path.open("w") as f:
1025 json.dump(index, f, sort_keys=True, indent=get_json_indent(d))
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index af6afcc653..19c6c0ff0b 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -3,1023 +3,6 @@
3# 3#
4# SPDX-License-Identifier: GPL-2.0-only 4# SPDX-License-Identifier: GPL-2.0-only
5# 5#
6 6# Include this class when you don't care what version of SPDX you get; it will
7DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${MACHINE}" 7# be updated to the latest stable version that is supported
8 8inherit create-spdx-2.2
9# The product name that the CVE database uses. Defaults to BPN, but may need to
10# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff).
11CVE_PRODUCT ??= "${BPN}"
12CVE_VERSION ??= "${PV}"
13
14SPDXDIR ??= "${WORKDIR}/spdx"
15SPDXDEPLOY = "${SPDXDIR}/deploy"
16SPDXWORK = "${SPDXDIR}/work"
17
18SPDX_TOOL_NAME ??= "oe-spdx-creator"
19SPDX_TOOL_VERSION ??= "1.0"
20
21SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy"
22
23SPDX_INCLUDE_SOURCES ??= "0"
24SPDX_ARCHIVE_SOURCES ??= "0"
25SPDX_ARCHIVE_PACKAGED ??= "0"
26
27SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org"
28SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc"
29SPDX_PRETTY ??= "0"
30
31SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
32
33SPDX_ORG ??= "OpenEmbedded ()"
34SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
35SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \
36 this recipe. For SPDX documents create using this class during the build, this \
37 is the contact information for the person or organization who is doing the \
38 build."
39
40def extract_licenses(filename):
41 import re
42
43 lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
44
45 try:
46 with open(filename, 'rb') as f:
47 size = min(15000, os.stat(filename).st_size)
48 txt = f.read(size)
49 licenses = re.findall(lic_regex, txt)
50 if licenses:
51 ascii_licenses = [lic.decode('ascii') for lic in licenses]
52 return ascii_licenses
53 except Exception as e:
54 bb.warn(f"Exception reading {filename}: {e}")
55 return None
56
57def get_doc_namespace(d, doc):
58 import uuid
59 namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
60 return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name)))
61
62def create_annotation(d, comment):
63 from datetime import datetime, timezone
64
65 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
66 annotation = oe.spdx.SPDXAnnotation()
67 annotation.annotationDate = creation_time
68 annotation.annotationType = "OTHER"
69 annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION"))
70 annotation.comment = comment
71 return annotation
72
73def recipe_spdx_is_native(d, recipe):
74 return any(a.annotationType == "OTHER" and
75 a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and
76 a.comment == "isNative" for a in recipe.annotations)
77
78def is_work_shared_spdx(d):
79 return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR'))
80
81def get_json_indent(d):
82 if d.getVar("SPDX_PRETTY") == "1":
83 return 2
84 return None
85
86python() {
87 import json
88 if d.getVar("SPDX_LICENSE_DATA"):
89 return
90
91 with open(d.getVar("SPDX_LICENSES"), "r") as f:
92 data = json.load(f)
93 # Transform the license array to a dictionary
94 data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
95 d.setVar("SPDX_LICENSE_DATA", data)
96}
97
98def convert_license_to_spdx(lic, document, d, existing={}):
99 from pathlib import Path
100 import oe.spdx
101
102 license_data = d.getVar("SPDX_LICENSE_DATA")
103 extracted = {}
104
105 def add_extracted_license(ident, name):
106 nonlocal document
107
108 if name in extracted:
109 return
110
111 extracted_info = oe.spdx.SPDXExtractedLicensingInfo()
112 extracted_info.name = name
113 extracted_info.licenseId = ident
114 extracted_info.extractedText = None
115
116 if name == "PD":
117 # Special-case this.
118 extracted_info.extractedText = "Software released to the public domain"
119 else:
120 # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH
121 for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split():
122 try:
123 with (Path(directory) / name).open(errors="replace") as f:
124 extracted_info.extractedText = f.read()
125 break
126 except FileNotFoundError:
127 pass
128 if extracted_info.extractedText is None:
129 # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set
130 filename = d.getVarFlag('NO_GENERIC_LICENSE', name)
131 if filename:
132 filename = d.expand("${S}/" + filename)
133 with open(filename, errors="replace") as f:
134 extracted_info.extractedText = f.read()
135 else:
136 bb.error("Cannot find any text for license %s" % name)
137
138 extracted[name] = extracted_info
139 document.hasExtractedLicensingInfos.append(extracted_info)
140
141 def convert(l):
142 if l == "(" or l == ")":
143 return l
144
145 if l == "&":
146 return "AND"
147
148 if l == "|":
149 return "OR"
150
151 if l == "CLOSED":
152 return "NONE"
153
154 spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l
155 if spdx_license in license_data["licenses"]:
156 return spdx_license
157
158 try:
159 spdx_license = existing[l]
160 except KeyError:
161 spdx_license = "LicenseRef-" + l
162 add_extracted_license(spdx_license, l)
163
164 return spdx_license
165
166 lic_split = lic.replace("(", " ( ").replace(")", " ) ").split()
167
168 return ' '.join(convert(l) for l in lic_split)
169
170def process_sources(d):
171 pn = d.getVar('PN')
172 assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
173 if pn in assume_provided:
174 for p in d.getVar("PROVIDES").split():
175 if p != pn:
176 pn = p
177 break
178
179 # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
180 # so avoid archiving source here.
181 if pn.startswith('glibc-locale'):
182 return False
183 if d.getVar('PN') == "libtool-cross":
184 return False
185 if d.getVar('PN') == "libgcc-initial":
186 return False
187 if d.getVar('PN') == "shadow-sysroot":
188 return False
189
190 # We just archive gcc-source for all the gcc related recipes
191 if d.getVar('BPN') in ['gcc', 'libgcc']:
192 bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
193 return False
194
195 return True
196
197
198def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]):
199 from pathlib import Path
200 import oe.spdx
201 import hashlib
202
203 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
204 if source_date_epoch:
205 source_date_epoch = int(source_date_epoch)
206
207 sha1s = []
208 spdx_files = []
209
210 file_counter = 1
211 for subdir, dirs, files in os.walk(topdir):
212 dirs[:] = [d for d in dirs if d not in ignore_dirs]
213 if subdir == str(topdir):
214 dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs]
215
216 for file in files:
217 filepath = Path(subdir) / file
218 filename = str(filepath.relative_to(topdir))
219
220 if not filepath.is_symlink() and filepath.is_file():
221 spdx_file = oe.spdx.SPDXFile()
222 spdx_file.SPDXID = get_spdxid(file_counter)
223 for t in get_types(filepath):
224 spdx_file.fileTypes.append(t)
225 spdx_file.fileName = filename
226
227 if archive is not None:
228 with filepath.open("rb") as f:
229 info = archive.gettarinfo(fileobj=f)
230 info.name = filename
231 info.uid = 0
232 info.gid = 0
233 info.uname = "root"
234 info.gname = "root"
235
236 if source_date_epoch is not None and info.mtime > source_date_epoch:
237 info.mtime = source_date_epoch
238
239 archive.addfile(info, f)
240
241 sha1 = bb.utils.sha1_file(filepath)
242 sha1s.append(sha1)
243 spdx_file.checksums.append(oe.spdx.SPDXChecksum(
244 algorithm="SHA1",
245 checksumValue=sha1,
246 ))
247 spdx_file.checksums.append(oe.spdx.SPDXChecksum(
248 algorithm="SHA256",
249 checksumValue=bb.utils.sha256_file(filepath),
250 ))
251
252 if "SOURCE" in spdx_file.fileTypes:
253 extracted_lics = extract_licenses(filepath)
254 if extracted_lics:
255 spdx_file.licenseInfoInFiles = extracted_lics
256
257 doc.files.append(spdx_file)
258 doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file)
259 spdx_pkg.hasFiles.append(spdx_file.SPDXID)
260
261 spdx_files.append(spdx_file)
262
263 file_counter += 1
264
265 sha1s.sort()
266 verifier = hashlib.sha1()
267 for v in sha1s:
268 verifier.update(v.encode("utf-8"))
269 spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest()
270
271 return spdx_files
272
273
274def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources):
275 from pathlib import Path
276 import hashlib
277 import oe.packagedata
278 import oe.spdx
279
280 debug_search_paths = [
281 Path(d.getVar('PKGD')),
282 Path(d.getVar('STAGING_DIR_TARGET')),
283 Path(d.getVar('STAGING_DIR_NATIVE')),
284 Path(d.getVar('STAGING_KERNEL_DIR')),
285 ]
286
287 pkg_data = oe.packagedata.read_subpkgdata_extended(package, d)
288
289 if pkg_data is None:
290 return
291
292 for file_path, file_data in pkg_data["files_info"].items():
293 if not "debugsrc" in file_data:
294 continue
295
296 for pkg_file in package_files:
297 if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"):
298 break
299 else:
300 bb.fatal("No package file found for %s" % str(file_path))
301 continue
302
303 for debugsrc in file_data["debugsrc"]:
304 ref_id = "NOASSERTION"
305 for search in debug_search_paths:
306 if debugsrc.startswith("/usr/src/kernel"):
307 debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '')
308 else:
309 debugsrc_path = search / debugsrc.lstrip("/")
310 if not debugsrc_path.exists():
311 continue
312
313 file_sha256 = bb.utils.sha256_file(debugsrc_path)
314
315 if file_sha256 in sources:
316 source_file = sources[file_sha256]
317
318 doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace)
319 if doc_ref is None:
320 doc_ref = oe.spdx.SPDXExternalDocumentRef()
321 doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name
322 doc_ref.spdxDocument = source_file.doc.documentNamespace
323 doc_ref.checksum.algorithm = "SHA1"
324 doc_ref.checksum.checksumValue = source_file.doc_sha1
325 package_doc.externalDocumentRefs.append(doc_ref)
326
327 ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID)
328 else:
329 bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256))
330 break
331 else:
332 bb.debug(1, "Debug source %s not found" % debugsrc)
333
334 package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc)
335
336def collect_dep_recipes(d, doc, spdx_recipe):
337 from pathlib import Path
338 import oe.sbom
339 import oe.spdx
340
341 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
342
343 dep_recipes = []
344 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
345 deps = sorted(set(
346 dep[0] for dep in taskdepdata.values() if
347 dep[1] == "do_create_spdx" and dep[0] != d.getVar("PN")
348 ))
349 for dep_pn in deps:
350 dep_recipe_path = deploy_dir_spdx / "recipes" / ("recipe-%s.spdx.json" % dep_pn)
351
352 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path)
353
354 for pkg in spdx_dep_doc.packages:
355 if pkg.name == dep_pn:
356 spdx_dep_recipe = pkg
357 break
358 else:
359 continue
360
361 dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe))
362
363 dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef()
364 dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name
365 dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace
366 dep_recipe_ref.checksum.algorithm = "SHA1"
367 dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1
368
369 doc.externalDocumentRefs.append(dep_recipe_ref)
370
371 doc.add_relationship(
372 "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID),
373 "BUILD_DEPENDENCY_OF",
374 spdx_recipe
375 )
376
377 return dep_recipes
378
379collect_dep_recipes[vardepsexclude] += "BB_TASKDEPDATA"
380
381
382def collect_dep_sources(d, dep_recipes):
383 import oe.sbom
384
385 sources = {}
386 for dep in dep_recipes:
387 # Don't collect sources from native recipes as they
388 # match non-native sources also.
389 if recipe_spdx_is_native(d, dep.recipe):
390 continue
391 recipe_files = set(dep.recipe.hasFiles)
392
393 for spdx_file in dep.doc.files:
394 if spdx_file.SPDXID not in recipe_files:
395 continue
396
397 if "SOURCE" in spdx_file.fileTypes:
398 for checksum in spdx_file.checksums:
399 if checksum.algorithm == "SHA256":
400 sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file)
401 break
402
403 return sources
404
405
406python do_create_spdx() {
407 from datetime import datetime, timezone
408 import oe.sbom
409 import oe.spdx
410 import uuid
411 from pathlib import Path
412 from contextlib import contextmanager
413 import oe.cve_check
414
415 @contextmanager
416 def optional_tarfile(name, guard, mode="w"):
417 import tarfile
418 import bb.compress.zstd
419
420 num_threads = int(d.getVar("BB_NUMBER_THREADS"))
421
422 if guard:
423 name.parent.mkdir(parents=True, exist_ok=True)
424 with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f:
425 with tarfile.open(fileobj=f, mode=mode + "|") as tf:
426 yield tf
427 else:
428 yield None
429
430
431 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
432 spdx_workdir = Path(d.getVar("SPDXWORK"))
433 include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
434 archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1"
435 archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1"
436
437 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
438
439 doc = oe.spdx.SPDXDocument()
440
441 doc.name = "recipe-" + d.getVar("PN")
442 doc.documentNamespace = get_doc_namespace(d, doc)
443 doc.creationInfo.created = creation_time
444 doc.creationInfo.comment = "This document was created by analyzing recipe files during the build."
445 doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
446 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
447 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
448 doc.creationInfo.creators.append("Person: N/A ()")
449
450 recipe = oe.spdx.SPDXPackage()
451 recipe.name = d.getVar("PN")
452 recipe.versionInfo = d.getVar("PV")
453 recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
454 recipe.supplier = d.getVar("SPDX_SUPPLIER")
455 if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d):
456 recipe.annotations.append(create_annotation(d, "isNative"))
457
458 for s in d.getVar('SRC_URI').split():
459 if not s.startswith("file://"):
460 s = s.split(';')[0]
461 recipe.downloadLocation = s
462 break
463 else:
464 recipe.downloadLocation = "NOASSERTION"
465
466 homepage = d.getVar("HOMEPAGE")
467 if homepage:
468 recipe.homepage = homepage
469
470 license = d.getVar("LICENSE")
471 if license:
472 recipe.licenseDeclared = convert_license_to_spdx(license, doc, d)
473
474 summary = d.getVar("SUMMARY")
475 if summary:
476 recipe.summary = summary
477
478 description = d.getVar("DESCRIPTION")
479 if description:
480 recipe.description = description
481
482 # Some CVEs may be patched during the build process without incrementing the version number,
483 # so querying for CVEs based on the CPE id can lead to false positives. To account for this,
484 # save the CVEs fixed by patches to source information field in the SPDX.
485 patched_cves = oe.cve_check.get_patched_cves(d)
486 patched_cves = list(patched_cves)
487 patched_cves = ' '.join(patched_cves)
488 if patched_cves:
489 recipe.sourceInfo = "CVEs fixed: " + patched_cves
490
491 cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION"))
492 if cpe_ids:
493 for cpe_id in cpe_ids:
494 cpe = oe.spdx.SPDXExternalReference()
495 cpe.referenceCategory = "SECURITY"
496 cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type"
497 cpe.referenceLocator = cpe_id
498 recipe.externalRefs.append(cpe)
499
500 doc.packages.append(recipe)
501 doc.add_relationship(doc, "DESCRIBES", recipe)
502
503 if process_sources(d) and include_sources:
504 recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst")
505 with optional_tarfile(recipe_archive, archive_sources) as archive:
506 spdx_get_src(d)
507
508 add_package_files(
509 d,
510 doc,
511 recipe,
512 spdx_workdir,
513 lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter),
514 lambda filepath: ["SOURCE"],
515 ignore_dirs=[".git"],
516 ignore_top_level_dirs=["temp"],
517 archive=archive,
518 )
519
520 if archive is not None:
521 recipe.packageFileName = str(recipe_archive.name)
522
523 dep_recipes = collect_dep_recipes(d, doc, recipe)
524
525 doc_sha1 = oe.sbom.write_doc(d, doc, "recipes", indent=get_json_indent(d))
526 dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe))
527
528 recipe_ref = oe.spdx.SPDXExternalDocumentRef()
529 recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name
530 recipe_ref.spdxDocument = doc.documentNamespace
531 recipe_ref.checksum.algorithm = "SHA1"
532 recipe_ref.checksum.checksumValue = doc_sha1
533
534 sources = collect_dep_sources(d, dep_recipes)
535 found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos}
536
537 if not recipe_spdx_is_native(d, recipe):
538 bb.build.exec_func("read_subpackage_metadata", d)
539
540 pkgdest = Path(d.getVar("PKGDEST"))
541 for package in d.getVar("PACKAGES").split():
542 if not oe.packagedata.packaged(package, d):
543 continue
544
545 package_doc = oe.spdx.SPDXDocument()
546 pkg_name = d.getVar("PKG:%s" % package) or package
547 package_doc.name = pkg_name
548 package_doc.documentNamespace = get_doc_namespace(d, package_doc)
549 package_doc.creationInfo.created = creation_time
550 package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build."
551 package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
552 package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
553 package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
554 package_doc.creationInfo.creators.append("Person: N/A ()")
555 package_doc.externalDocumentRefs.append(recipe_ref)
556
557 package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE")
558
559 spdx_package = oe.spdx.SPDXPackage()
560
561 spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name)
562 spdx_package.name = pkg_name
563 spdx_package.versionInfo = d.getVar("PV")
564 spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses)
565 spdx_package.supplier = d.getVar("SPDX_SUPPLIER")
566
567 package_doc.packages.append(spdx_package)
568
569 package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID))
570 package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package)
571
572 package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst")
573 with optional_tarfile(package_archive, archive_packaged) as archive:
574 package_files = add_package_files(
575 d,
576 package_doc,
577 spdx_package,
578 pkgdest / package,
579 lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter),
580 lambda filepath: ["BINARY"],
581 ignore_top_level_dirs=['CONTROL', 'DEBIAN'],
582 archive=archive,
583 )
584
585 if archive is not None:
586 spdx_package.packageFileName = str(package_archive.name)
587
588 add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources)
589
590 oe.sbom.write_doc(d, package_doc, "packages", indent=get_json_indent(d))
591}
592# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
593addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work
594
595SSTATETASKS += "do_create_spdx"
596do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
597do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
598
599python do_create_spdx_setscene () {
600 sstate_setscene(d)
601}
602addtask do_create_spdx_setscene
603
604do_create_spdx[dirs] = "${SPDXWORK}"
605do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}"
606do_create_spdx[depends] += "${PATCHDEPENDENCY}"
607do_create_spdx[deptask] = "do_create_spdx"
608
609def collect_package_providers(d):
610 from pathlib import Path
611 import oe.sbom
612 import oe.spdx
613 import json
614
615 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
616
617 providers = {}
618
619 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
620 deps = sorted(set(
621 dep[0] for dep in taskdepdata.values() if dep[0] != d.getVar("PN")
622 ))
623 deps.append(d.getVar("PN"))
624
625 for dep_pn in deps:
626 recipe_data = oe.packagedata.read_pkgdata(dep_pn, d)
627
628 for pkg in recipe_data.get("PACKAGES", "").split():
629
630 pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, d)
631 rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items())
632 rprovides.add(pkg)
633
634 for r in rprovides:
635 providers[r] = pkg
636
637 return providers
638
639collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA"
640
641python do_create_runtime_spdx() {
642 from datetime import datetime, timezone
643 import oe.sbom
644 import oe.spdx
645 import oe.packagedata
646 from pathlib import Path
647
648 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
649 spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY"))
650 is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d)
651
652 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
653
654 providers = collect_package_providers(d)
655
656 if not is_native:
657 bb.build.exec_func("read_subpackage_metadata", d)
658
659 dep_package_cache = {}
660
661 pkgdest = Path(d.getVar("PKGDEST"))
662 for package in d.getVar("PACKAGES").split():
663 localdata = bb.data.createCopy(d)
664 pkg_name = d.getVar("PKG:%s" % package) or package
665 localdata.setVar("PKG", pkg_name)
666 localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package)
667
668 if not oe.packagedata.packaged(package, localdata):
669 continue
670
671 pkg_spdx_path = deploy_dir_spdx / "packages" / (pkg_name + ".spdx.json")
672
673 package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path)
674
675 for p in package_doc.packages:
676 if p.name == pkg_name:
677 spdx_package = p
678 break
679 else:
680 bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path))
681
682 runtime_doc = oe.spdx.SPDXDocument()
683 runtime_doc.name = "runtime-" + pkg_name
684 runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc)
685 runtime_doc.creationInfo.created = creation_time
686 runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies."
687 runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
688 runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
689 runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
690 runtime_doc.creationInfo.creators.append("Person: N/A ()")
691
692 package_ref = oe.spdx.SPDXExternalDocumentRef()
693 package_ref.externalDocumentId = "DocumentRef-package-" + package
694 package_ref.spdxDocument = package_doc.documentNamespace
695 package_ref.checksum.algorithm = "SHA1"
696 package_ref.checksum.checksumValue = package_doc_sha1
697
698 runtime_doc.externalDocumentRefs.append(package_ref)
699
700 runtime_doc.add_relationship(
701 runtime_doc.SPDXID,
702 "AMENDS",
703 "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID)
704 )
705
706 deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "")
707 seen_deps = set()
708 for dep, _ in deps.items():
709 if dep in seen_deps:
710 continue
711
712 if dep not in providers:
713 continue
714
715 dep = providers[dep]
716
717 if not oe.packagedata.packaged(dep, localdata):
718 continue
719
720 dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d)
721 dep_pkg = dep_pkg_data["PKG"]
722
723 if dep in dep_package_cache:
724 (dep_spdx_package, dep_package_ref) = dep_package_cache[dep]
725 else:
726 dep_path = deploy_dir_spdx / "packages" / ("%s.spdx.json" % dep_pkg)
727
728 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path)
729
730 for pkg in spdx_dep_doc.packages:
731 if pkg.name == dep_pkg:
732 dep_spdx_package = pkg
733 break
734 else:
735 bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path))
736
737 dep_package_ref = oe.spdx.SPDXExternalDocumentRef()
738 dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name
739 dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace
740 dep_package_ref.checksum.algorithm = "SHA1"
741 dep_package_ref.checksum.checksumValue = spdx_dep_sha1
742
743 dep_package_cache[dep] = (dep_spdx_package, dep_package_ref)
744
745 runtime_doc.externalDocumentRefs.append(dep_package_ref)
746
747 runtime_doc.add_relationship(
748 "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID),
749 "RUNTIME_DEPENDENCY_OF",
750 "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID)
751 )
752 seen_deps.add(dep)
753
754 oe.sbom.write_doc(d, runtime_doc, "runtime", spdx_deploy, indent=get_json_indent(d))
755}
756
757addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work
758SSTATETASKS += "do_create_runtime_spdx"
759do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}"
760do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
761
762python do_create_runtime_spdx_setscene () {
763 sstate_setscene(d)
764}
765addtask do_create_runtime_spdx_setscene
766
767do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}"
768do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}"
769do_create_runtime_spdx[rdeptask] = "do_create_spdx"
770
771def spdx_get_src(d):
772 """
773 save patched source of the recipe in SPDX_WORKDIR.
774 """
775 import shutil
776 spdx_workdir = d.getVar('SPDXWORK')
777 spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE')
778 pn = d.getVar('PN')
779
780 workdir = d.getVar("WORKDIR")
781
782 try:
783 # The kernel class functions require it to be on work-shared, so we dont change WORKDIR
784 if not is_work_shared_spdx(d):
785 # Change the WORKDIR to make do_unpack do_patch run in another dir.
786 d.setVar('WORKDIR', spdx_workdir)
787 # Restore the original path to recipe's native sysroot (it's relative to WORKDIR).
788 d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)
789
790 # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the
791 # possibly requiring of the following tasks (such as some recipes's
792 # do_patch required 'B' existed).
793 bb.utils.mkdirhier(d.getVar('B'))
794
795 bb.build.exec_func('do_unpack', d)
796 # Copy source of kernel to spdx_workdir
797 if is_work_shared_spdx(d):
798 d.setVar('WORKDIR', spdx_workdir)
799 d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)
800 src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR')
801 bb.utils.mkdirhier(src_dir)
802 if bb.data.inherits_class('kernel',d):
803 share_src = d.getVar('STAGING_KERNEL_DIR')
804 cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/"
805 cmd_copy_kernel_result = os.popen(cmd_copy_share).read()
806 bb.note("cmd_copy_kernel_result = " + cmd_copy_kernel_result)
807
808 git_path = src_dir + "/.git"
809 if os.path.exists(git_path):
810 shutils.rmtree(git_path)
811
812 # Make sure gcc and kernel sources are patched only once
813 if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)):
814 bb.build.exec_func('do_patch', d)
815
816 # Some userland has no source.
817 if not os.path.exists( spdx_workdir ):
818 bb.utils.mkdirhier(spdx_workdir)
819 finally:
820 d.setVar("WORKDIR", workdir)
821
822do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
823
824ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; "
825
826do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
827POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx; "
828POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx; "
829
830python image_combine_spdx() {
831 import os
832 import oe.sbom
833 from pathlib import Path
834 from oe.rootfs import image_list_installed_packages
835
836 image_name = d.getVar("IMAGE_NAME")
837 image_link_name = d.getVar("IMAGE_LINK_NAME")
838 imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
839 img_spdxid = oe.sbom.get_image_spdxid(image_name)
840 packages = image_list_installed_packages(d)
841
842 combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages)
843
844 def make_image_link(target_path, suffix):
845 if image_link_name:
846 link = imgdeploydir / (image_link_name + suffix)
847 if link != target_path:
848 link.symlink_to(os.path.relpath(target_path, link.parent))
849
850 image_spdx_path = imgdeploydir / (image_name + ".spdx.json")
851 make_image_link(image_spdx_path, ".spdx.json")
852 spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst")
853 make_image_link(spdx_tar_path, ".spdx.tar.zst")
854 spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json")
855 make_image_link(spdx_index_path, ".spdx.index.json")
856}
857
858python sdk_host_combine_spdx() {
859 sdk_combine_spdx(d, "host")
860}
861
862python sdk_target_combine_spdx() {
863 sdk_combine_spdx(d, "target")
864}
865
866def sdk_combine_spdx(d, sdk_type):
867 import oe.sbom
868 from pathlib import Path
869 from oe.sdk import sdk_list_installed_packages
870
871 sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type
872 sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR"))
873 sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name)
874 sdk_packages = sdk_list_installed_packages(d, sdk_type == "target")
875 combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages)
876
877def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages):
878 import os
879 import oe.spdx
880 import oe.sbom
881 import io
882 import json
883 from datetime import timezone, datetime
884 from pathlib import Path
885 import tarfile
886 import bb.compress.zstd
887
888 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
889 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
890 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
891
892 doc = oe.spdx.SPDXDocument()
893 doc.name = rootfs_name
894 doc.documentNamespace = get_doc_namespace(d, doc)
895 doc.creationInfo.created = creation_time
896 doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build."
897 doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
898 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
899 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
900 doc.creationInfo.creators.append("Person: N/A ()")
901
902 image = oe.spdx.SPDXPackage()
903 image.name = d.getVar("PN")
904 image.versionInfo = d.getVar("PV")
905 image.SPDXID = rootfs_spdxid
906 image.supplier = d.getVar("SPDX_SUPPLIER")
907
908 doc.packages.append(image)
909
910 for name in sorted(packages.keys()):
911 pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json")
912 pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path)
913
914 for p in pkg_doc.packages:
915 if p.name == name:
916 pkg_ref = oe.spdx.SPDXExternalDocumentRef()
917 pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name
918 pkg_ref.spdxDocument = pkg_doc.documentNamespace
919 pkg_ref.checksum.algorithm = "SHA1"
920 pkg_ref.checksum.checksumValue = pkg_doc_sha1
921
922 doc.externalDocumentRefs.append(pkg_ref)
923 doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID))
924 break
925 else:
926 bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path))
927
928 runtime_spdx_path = deploy_dir_spdx / "runtime" / ("runtime-" + name + ".spdx.json")
929 runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path)
930
931 runtime_ref = oe.spdx.SPDXExternalDocumentRef()
932 runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name
933 runtime_ref.spdxDocument = runtime_doc.documentNamespace
934 runtime_ref.checksum.algorithm = "SHA1"
935 runtime_ref.checksum.checksumValue = runtime_doc_sha1
936
937 # "OTHER" isn't ideal here, but I can't find a relationship that makes sense
938 doc.externalDocumentRefs.append(runtime_ref)
939 doc.add_relationship(
940 image,
941 "OTHER",
942 "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID),
943 comment="Runtime dependencies for %s" % name
944 )
945
946 image_spdx_path = rootfs_deploydir / (rootfs_name + ".spdx.json")
947
948 with image_spdx_path.open("wb") as f:
949 doc.to_json(f, sort_keys=True, indent=get_json_indent(d))
950
951 num_threads = int(d.getVar("BB_NUMBER_THREADS"))
952
953 visited_docs = set()
954
955 index = {"documents": []}
956
957 spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst")
958 with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f:
959 with tarfile.open(fileobj=f, mode="w|") as tar:
960 def collect_spdx_document(path):
961 nonlocal tar
962 nonlocal deploy_dir_spdx
963 nonlocal source_date_epoch
964 nonlocal index
965
966 if path in visited_docs:
967 return
968
969 visited_docs.add(path)
970
971 with path.open("rb") as f:
972 doc, sha1 = oe.sbom.read_doc(f)
973 f.seek(0)
974
975 if doc.documentNamespace in visited_docs:
976 return
977
978 bb.note("Adding SPDX document %s" % path)
979 visited_docs.add(doc.documentNamespace)
980 info = tar.gettarinfo(fileobj=f)
981
982 info.name = doc.name + ".spdx.json"
983 info.uid = 0
984 info.gid = 0
985 info.uname = "root"
986 info.gname = "root"
987
988 if source_date_epoch is not None and info.mtime > int(source_date_epoch):
989 info.mtime = int(source_date_epoch)
990
991 tar.addfile(info, f)
992
993 index["documents"].append({
994 "filename": info.name,
995 "documentNamespace": doc.documentNamespace,
996 "sha1": sha1,
997 })
998
999 for ref in doc.externalDocumentRefs:
1000 ref_path = deploy_dir_spdx / "by-namespace" / ref.spdxDocument.replace("/", "_")
1001 collect_spdx_document(ref_path)
1002
1003 collect_spdx_document(image_spdx_path)
1004
1005 index["documents"].sort(key=lambda x: x["filename"])
1006
1007 index_str = io.BytesIO(json.dumps(
1008 index,
1009 sort_keys=True,
1010 indent=get_json_indent(d),
1011 ).encode("utf-8"))
1012
1013 info = tarfile.TarInfo()
1014 info.name = "index.json"
1015 info.size = len(index_str.getvalue())
1016 info.uid = 0
1017 info.gid = 0
1018 info.uname = "root"
1019 info.gname = "root"
1020
1021 tar.addfile(info, fileobj=index_str)
1022
1023 spdx_index_path = rootfs_deploydir / (rootfs_name + ".spdx.index.json")
1024 with spdx_index_path.open("w") as f:
1025 json.dump(index, f, sort_keys=True, indent=get_json_indent(d))