summaryrefslogtreecommitdiffstats
path: root/meta/classes/create-spdx-2.2.bbclass
diff options
context:
space:
mode:
Diffstat (limited to 'meta/classes/create-spdx-2.2.bbclass')
-rw-r--r--meta/classes/create-spdx-2.2.bbclass1067
1 files changed, 1067 insertions, 0 deletions
diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass
new file mode 100644
index 0000000000..42b693d586
--- /dev/null
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -0,0 +1,1067 @@
1#
2# Copyright OpenEmbedded Contributors
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6
7DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${MACHINE}"
8
9# The product name that the CVE database uses. Defaults to BPN, but may need to
10# be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff).
11CVE_PRODUCT ??= "${BPN}"
12CVE_VERSION ??= "${PV}"
13
14SPDXDIR ??= "${WORKDIR}/spdx"
15SPDXDEPLOY = "${SPDXDIR}/deploy"
16SPDXWORK = "${SPDXDIR}/work"
17SPDXIMAGEWORK = "${SPDXDIR}/image-work"
18SPDXSDKWORK = "${SPDXDIR}/sdk-work"
19
20SPDX_TOOL_NAME ??= "oe-spdx-creator"
21SPDX_TOOL_VERSION ??= "1.0"
22
23SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy"
24
25SPDX_INCLUDE_SOURCES ??= "0"
26SPDX_ARCHIVE_SOURCES ??= "0"
27SPDX_ARCHIVE_PACKAGED ??= "0"
28
29SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org"
30SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc"
31SPDX_PRETTY ??= "0"
32
33SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
34
35SPDX_CUSTOM_ANNOTATION_VARS ??= ""
36
37SPDX_ORG ??= "OpenEmbedded ()"
38SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
39SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \
40 this recipe. For SPDX documents create using this class during the build, this \
41 is the contact information for the person or organization who is doing the \
42 build."
43
44def extract_licenses(filename):
45 import re
46
47 lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
48
49 try:
50 with open(filename, 'rb') as f:
51 size = min(15000, os.stat(filename).st_size)
52 txt = f.read(size)
53 licenses = re.findall(lic_regex, txt)
54 if licenses:
55 ascii_licenses = [lic.decode('ascii') for lic in licenses]
56 return ascii_licenses
57 except Exception as e:
58 bb.warn(f"Exception reading {filename}: {e}")
59 return None
60
61def get_doc_namespace(d, doc):
62 import uuid
63 namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE"))
64 return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name)))
65
66def create_annotation(d, comment):
67 from datetime import datetime, timezone
68
69 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
70 annotation = oe.spdx.SPDXAnnotation()
71 annotation.annotationDate = creation_time
72 annotation.annotationType = "OTHER"
73 annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION"))
74 annotation.comment = comment
75 return annotation
76
77def recipe_spdx_is_native(d, recipe):
78 return any(a.annotationType == "OTHER" and
79 a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and
80 a.comment == "isNative" for a in recipe.annotations)
81
82def is_work_shared_spdx(d):
83 return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR'))
84
85def get_json_indent(d):
86 if d.getVar("SPDX_PRETTY") == "1":
87 return 2
88 return None
89
90python() {
91 import json
92 if d.getVar("SPDX_LICENSE_DATA"):
93 return
94
95 with open(d.getVar("SPDX_LICENSES"), "r") as f:
96 data = json.load(f)
97 # Transform the license array to a dictionary
98 data["licenses"] = {l["licenseId"]: l for l in data["licenses"]}
99 d.setVar("SPDX_LICENSE_DATA", data)
100}
101
102def convert_license_to_spdx(lic, document, d, existing={}):
103 from pathlib import Path
104 import oe.spdx
105
106 license_data = d.getVar("SPDX_LICENSE_DATA")
107 extracted = {}
108
109 def add_extracted_license(ident, name):
110 nonlocal document
111
112 if name in extracted:
113 return
114
115 extracted_info = oe.spdx.SPDXExtractedLicensingInfo()
116 extracted_info.name = name
117 extracted_info.licenseId = ident
118 extracted_info.extractedText = None
119
120 if name == "PD":
121 # Special-case this.
122 extracted_info.extractedText = "Software released to the public domain"
123 else:
124 # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH
125 for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split():
126 try:
127 with (Path(directory) / name).open(errors="replace") as f:
128 extracted_info.extractedText = f.read()
129 break
130 except FileNotFoundError:
131 pass
132 if extracted_info.extractedText is None:
133 # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set
134 filename = d.getVarFlag('NO_GENERIC_LICENSE', name)
135 if filename:
136 filename = d.expand("${S}/" + filename)
137 with open(filename, errors="replace") as f:
138 extracted_info.extractedText = f.read()
139 else:
140 bb.error("Cannot find any text for license %s" % name)
141
142 extracted[name] = extracted_info
143 document.hasExtractedLicensingInfos.append(extracted_info)
144
145 def convert(l):
146 if l == "(" or l == ")":
147 return l
148
149 if l == "&":
150 return "AND"
151
152 if l == "|":
153 return "OR"
154
155 if l == "CLOSED":
156 return "NONE"
157
158 spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l
159 if spdx_license in license_data["licenses"]:
160 return spdx_license
161
162 try:
163 spdx_license = existing[l]
164 except KeyError:
165 spdx_license = "LicenseRef-" + l
166 add_extracted_license(spdx_license, l)
167
168 return spdx_license
169
170 lic_split = lic.replace("(", " ( ").replace(")", " ) ").split()
171
172 return ' '.join(convert(l) for l in lic_split)
173
174def process_sources(d):
175 pn = d.getVar('PN')
176 assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
177 if pn in assume_provided:
178 for p in d.getVar("PROVIDES").split():
179 if p != pn:
180 pn = p
181 break
182
183 # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted,
184 # so avoid archiving source here.
185 if pn.startswith('glibc-locale'):
186 return False
187 if d.getVar('PN') == "libtool-cross":
188 return False
189 if d.getVar('PN') == "libgcc-initial":
190 return False
191 if d.getVar('PN') == "shadow-sysroot":
192 return False
193
194 # We just archive gcc-source for all the gcc related recipes
195 if d.getVar('BPN') in ['gcc', 'libgcc']:
196 bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn)
197 return False
198
199 return True
200
201
202def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]):
203 from pathlib import Path
204 import oe.spdx
205 import hashlib
206
207 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
208 if source_date_epoch:
209 source_date_epoch = int(source_date_epoch)
210
211 sha1s = []
212 spdx_files = []
213
214 file_counter = 1
215 for subdir, dirs, files in os.walk(topdir):
216 dirs[:] = [d for d in dirs if d not in ignore_dirs]
217 if subdir == str(topdir):
218 dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs]
219
220 for file in files:
221 filepath = Path(subdir) / file
222 filename = str(filepath.relative_to(topdir))
223
224 if not filepath.is_symlink() and filepath.is_file():
225 spdx_file = oe.spdx.SPDXFile()
226 spdx_file.SPDXID = get_spdxid(file_counter)
227 for t in get_types(filepath):
228 spdx_file.fileTypes.append(t)
229 spdx_file.fileName = filename
230
231 if archive is not None:
232 with filepath.open("rb") as f:
233 info = archive.gettarinfo(fileobj=f)
234 info.name = filename
235 info.uid = 0
236 info.gid = 0
237 info.uname = "root"
238 info.gname = "root"
239
240 if source_date_epoch is not None and info.mtime > source_date_epoch:
241 info.mtime = source_date_epoch
242
243 archive.addfile(info, f)
244
245 sha1 = bb.utils.sha1_file(filepath)
246 sha1s.append(sha1)
247 spdx_file.checksums.append(oe.spdx.SPDXChecksum(
248 algorithm="SHA1",
249 checksumValue=sha1,
250 ))
251 spdx_file.checksums.append(oe.spdx.SPDXChecksum(
252 algorithm="SHA256",
253 checksumValue=bb.utils.sha256_file(filepath),
254 ))
255
256 if "SOURCE" in spdx_file.fileTypes:
257 extracted_lics = extract_licenses(filepath)
258 if extracted_lics:
259 spdx_file.licenseInfoInFiles = extracted_lics
260
261 doc.files.append(spdx_file)
262 doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file)
263 spdx_pkg.hasFiles.append(spdx_file.SPDXID)
264
265 spdx_files.append(spdx_file)
266
267 file_counter += 1
268
269 sha1s.sort()
270 verifier = hashlib.sha1()
271 for v in sha1s:
272 verifier.update(v.encode("utf-8"))
273 spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest()
274
275 return spdx_files
276
277
278def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources):
279 from pathlib import Path
280 import hashlib
281 import oe.packagedata
282 import oe.spdx
283
284 debug_search_paths = [
285 Path(d.getVar('PKGD')),
286 Path(d.getVar('STAGING_DIR_TARGET')),
287 Path(d.getVar('STAGING_DIR_NATIVE')),
288 Path(d.getVar('STAGING_KERNEL_DIR')),
289 ]
290
291 pkg_data = oe.packagedata.read_subpkgdata_extended(package, d)
292
293 if pkg_data is None:
294 return
295
296 for file_path, file_data in pkg_data["files_info"].items():
297 if not "debugsrc" in file_data:
298 continue
299
300 for pkg_file in package_files:
301 if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"):
302 break
303 else:
304 bb.fatal("No package file found for %s" % str(file_path))
305 continue
306
307 for debugsrc in file_data["debugsrc"]:
308 ref_id = "NOASSERTION"
309 for search in debug_search_paths:
310 if debugsrc.startswith("/usr/src/kernel"):
311 debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '')
312 else:
313 debugsrc_path = search / debugsrc.lstrip("/")
314 if not debugsrc_path.exists():
315 continue
316
317 file_sha256 = bb.utils.sha256_file(debugsrc_path)
318
319 if file_sha256 in sources:
320 source_file = sources[file_sha256]
321
322 doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace)
323 if doc_ref is None:
324 doc_ref = oe.spdx.SPDXExternalDocumentRef()
325 doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name
326 doc_ref.spdxDocument = source_file.doc.documentNamespace
327 doc_ref.checksum.algorithm = "SHA1"
328 doc_ref.checksum.checksumValue = source_file.doc_sha1
329 package_doc.externalDocumentRefs.append(doc_ref)
330
331 ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID)
332 else:
333 bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256))
334 break
335 else:
336 bb.debug(1, "Debug source %s not found" % debugsrc)
337
338 package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc)
339
340def collect_dep_recipes(d, doc, spdx_recipe):
341 from pathlib import Path
342 import oe.sbom
343 import oe.spdx
344
345 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
346
347 dep_recipes = []
348 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
349 deps = sorted(set(
350 dep[0] for dep in taskdepdata.values() if
351 dep[1] == "do_create_spdx" and dep[0] != d.getVar("PN")
352 ))
353 for dep_pn in deps:
354 dep_recipe_path = deploy_dir_spdx / "recipes" / ("recipe-%s.spdx.json" % dep_pn)
355
356 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path)
357
358 for pkg in spdx_dep_doc.packages:
359 if pkg.name == dep_pn:
360 spdx_dep_recipe = pkg
361 break
362 else:
363 continue
364
365 dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe))
366
367 dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef()
368 dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name
369 dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace
370 dep_recipe_ref.checksum.algorithm = "SHA1"
371 dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1
372
373 doc.externalDocumentRefs.append(dep_recipe_ref)
374
375 doc.add_relationship(
376 "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID),
377 "BUILD_DEPENDENCY_OF",
378 spdx_recipe
379 )
380
381 return dep_recipes
382
383collect_dep_recipes[vardepsexclude] += "BB_TASKDEPDATA"
384collect_dep_recipes[vardeps] += "DEPENDS"
385
386def collect_dep_sources(d, dep_recipes):
387 import oe.sbom
388
389 sources = {}
390 for dep in dep_recipes:
391 # Don't collect sources from native recipes as they
392 # match non-native sources also.
393 if recipe_spdx_is_native(d, dep.recipe):
394 continue
395 recipe_files = set(dep.recipe.hasFiles)
396
397 for spdx_file in dep.doc.files:
398 if spdx_file.SPDXID not in recipe_files:
399 continue
400
401 if "SOURCE" in spdx_file.fileTypes:
402 for checksum in spdx_file.checksums:
403 if checksum.algorithm == "SHA256":
404 sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file)
405 break
406
407 return sources
408
409def add_download_packages(d, doc, recipe):
410 import os.path
411 from bb.fetch2 import decodeurl, CHECKSUM_LIST
412 import bb.process
413 import oe.spdx
414 import oe.sbom
415
416 for download_idx, src_uri in enumerate(d.getVar('SRC_URI').split()):
417 f = bb.fetch2.FetchData(src_uri, d)
418
419 for name in f.names:
420 package = oe.spdx.SPDXPackage()
421 package.name = "%s-source-%d" % (d.getVar("PN"), download_idx + 1)
422 package.SPDXID = oe.sbom.get_download_spdxid(d, download_idx + 1)
423
424 if f.type == "file":
425 continue
426
427 uri = f.type
428 proto = getattr(f, "proto", None)
429 if proto is not None:
430 uri = uri + "+" + proto
431 uri = uri + "://" + f.host + f.path
432
433 if f.method.supports_srcrev():
434 uri = uri + "@" + f.revisions[name]
435
436 if f.method.supports_checksum(f):
437 for checksum_id in CHECKSUM_LIST:
438 if checksum_id.upper() not in oe.spdx.SPDXPackage.ALLOWED_CHECKSUMS:
439 continue
440
441 expected_checksum = getattr(f, "%s_expected" % checksum_id)
442 if expected_checksum is None:
443 continue
444
445 c = oe.spdx.SPDXChecksum()
446 c.algorithm = checksum_id.upper()
447 c.checksumValue = expected_checksum
448 package.checksums.append(c)
449
450 package.downloadLocation = uri
451 doc.packages.append(package)
452 doc.add_relationship(doc, "DESCRIBES", package)
453 # In the future, we might be able to do more fancy dependencies,
454 # but this should be sufficient for now
455 doc.add_relationship(package, "BUILD_DEPENDENCY_OF", recipe)
456
457python do_create_spdx() {
458 from datetime import datetime, timezone
459 import oe.sbom
460 import oe.spdx
461 import uuid
462 from pathlib import Path
463 from contextlib import contextmanager
464 import oe.cve_check
465
466 @contextmanager
467 def optional_tarfile(name, guard, mode="w"):
468 import tarfile
469 import gzip
470
471 if guard:
472 name.parent.mkdir(parents=True, exist_ok=True)
473 with gzip.open(name, mode=mode + "b") as f:
474 with tarfile.open(fileobj=f, mode=mode + "|") as tf:
475 yield tf
476 else:
477 yield None
478
479
480 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
481 spdx_workdir = Path(d.getVar("SPDXWORK"))
482 include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
483 archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1"
484 archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1"
485
486 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
487
488 doc = oe.spdx.SPDXDocument()
489
490 doc.name = "recipe-" + d.getVar("PN")
491 doc.documentNamespace = get_doc_namespace(d, doc)
492 doc.creationInfo.created = creation_time
493 doc.creationInfo.comment = "This document was created by analyzing recipe files during the build."
494 doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
495 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
496 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
497 doc.creationInfo.creators.append("Person: N/A ()")
498
499 recipe = oe.spdx.SPDXPackage()
500 recipe.name = d.getVar("PN")
501 recipe.versionInfo = d.getVar("PV")
502 recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
503 recipe.supplier = d.getVar("SPDX_SUPPLIER")
504 if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d):
505 recipe.annotations.append(create_annotation(d, "isNative"))
506
507 homepage = d.getVar("HOMEPAGE")
508 if homepage:
509 recipe.homepage = homepage
510
511 license = d.getVar("LICENSE")
512 if license:
513 recipe.licenseDeclared = convert_license_to_spdx(license, doc, d)
514
515 summary = d.getVar("SUMMARY")
516 if summary:
517 recipe.summary = summary
518
519 description = d.getVar("DESCRIPTION")
520 if description:
521 recipe.description = description
522
523 if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"):
524 for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split():
525 recipe.annotations.append(create_annotation(d, var + "=" + d.getVar(var)))
526
527 # Some CVEs may be patched during the build process without incrementing the version number,
528 # so querying for CVEs based on the CPE id can lead to false positives. To account for this,
529 # save the CVEs fixed by patches to source information field in the SPDX.
530 patched_cves = oe.cve_check.get_patched_cves(d)
531 patched_cves = list(patched_cves)
532 patched_cves = ' '.join(patched_cves)
533 if patched_cves:
534 recipe.sourceInfo = "CVEs fixed: " + patched_cves
535
536 cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION"))
537 if cpe_ids:
538 for cpe_id in cpe_ids:
539 cpe = oe.spdx.SPDXExternalReference()
540 cpe.referenceCategory = "SECURITY"
541 cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type"
542 cpe.referenceLocator = cpe_id
543 recipe.externalRefs.append(cpe)
544
545 doc.packages.append(recipe)
546 doc.add_relationship(doc, "DESCRIBES", recipe)
547
548 add_download_packages(d, doc, recipe)
549
550 if process_sources(d) and include_sources:
551 recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.gz")
552 with optional_tarfile(recipe_archive, archive_sources) as archive:
553 spdx_get_src(d)
554
555 add_package_files(
556 d,
557 doc,
558 recipe,
559 spdx_workdir,
560 lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter),
561 lambda filepath: ["SOURCE"],
562 ignore_dirs=[".git"],
563 ignore_top_level_dirs=["temp"],
564 archive=archive,
565 )
566
567 if archive is not None:
568 recipe.packageFileName = str(recipe_archive.name)
569
570 dep_recipes = collect_dep_recipes(d, doc, recipe)
571
572 doc_sha1 = oe.sbom.write_doc(d, doc, "recipes", indent=get_json_indent(d))
573 dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe))
574
575 recipe_ref = oe.spdx.SPDXExternalDocumentRef()
576 recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name
577 recipe_ref.spdxDocument = doc.documentNamespace
578 recipe_ref.checksum.algorithm = "SHA1"
579 recipe_ref.checksum.checksumValue = doc_sha1
580
581 sources = collect_dep_sources(d, dep_recipes)
582 found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos}
583
584 if not recipe_spdx_is_native(d, recipe):
585 bb.build.exec_func("read_subpackage_metadata", d)
586
587 pkgdest = Path(d.getVar("PKGDEST"))
588 for package in d.getVar("PACKAGES").split():
589 if not oe.packagedata.packaged(package, d):
590 continue
591
592 package_doc = oe.spdx.SPDXDocument()
593 pkg_name = d.getVar("PKG:%s" % package) or package
594 package_doc.name = pkg_name
595 package_doc.documentNamespace = get_doc_namespace(d, package_doc)
596 package_doc.creationInfo.created = creation_time
597 package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build."
598 package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
599 package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
600 package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
601 package_doc.creationInfo.creators.append("Person: N/A ()")
602 package_doc.externalDocumentRefs.append(recipe_ref)
603
604 package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE")
605
606 spdx_package = oe.spdx.SPDXPackage()
607
608 spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name)
609 spdx_package.name = pkg_name
610 spdx_package.versionInfo = d.getVar("PV")
611 spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses)
612 spdx_package.supplier = d.getVar("SPDX_SUPPLIER")
613
614 package_doc.packages.append(spdx_package)
615
616 package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID))
617 package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package)
618
619 package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.gz")
620 with optional_tarfile(package_archive, archive_packaged) as archive:
621 package_files = add_package_files(
622 d,
623 package_doc,
624 spdx_package,
625 pkgdest / package,
626 lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter),
627 lambda filepath: ["BINARY"],
628 ignore_top_level_dirs=['CONTROL', 'DEBIAN'],
629 archive=archive,
630 )
631
632 if archive is not None:
633 spdx_package.packageFileName = str(package_archive.name)
634
635 add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources)
636
637 oe.sbom.write_doc(d, package_doc, "packages", indent=get_json_indent(d))
638}
639# NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source
640addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work
641
642SSTATETASKS += "do_create_spdx"
643do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
644do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
645
646python do_create_spdx_setscene () {
647 sstate_setscene(d)
648}
649addtask do_create_spdx_setscene
650
651do_create_spdx[dirs] = "${SPDXWORK}"
652do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}"
653do_create_spdx[depends] += "${PATCHDEPENDENCY}"
654do_create_spdx[deptask] = "do_create_spdx"
655
656def collect_package_providers(d):
657 from pathlib import Path
658 import oe.sbom
659 import oe.spdx
660 import json
661
662 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
663
664 providers = {}
665
666 taskdepdata = d.getVar("BB_TASKDEPDATA", False)
667 deps = sorted(set(
668 dep[0] for dep in taskdepdata.values() if dep[0] != d.getVar("PN")
669 ))
670 deps.append(d.getVar("PN"))
671
672 for dep_pn in deps:
673 recipe_data = oe.packagedata.read_pkgdata(dep_pn, d)
674
675 for pkg in recipe_data.get("PACKAGES", "").split():
676
677 pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, d)
678 rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items())
679 rprovides.add(pkg)
680
681 for r in rprovides:
682 providers[r] = pkg
683
684 return providers
685
686collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA"
687
688python do_create_runtime_spdx() {
689 from datetime import datetime, timezone
690 import oe.sbom
691 import oe.spdx
692 import oe.packagedata
693 from pathlib import Path
694
695 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
696 spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY"))
697 is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d)
698
699 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
700
701 providers = collect_package_providers(d)
702
703 if not is_native:
704 bb.build.exec_func("read_subpackage_metadata", d)
705
706 dep_package_cache = {}
707
708 pkgdest = Path(d.getVar("PKGDEST"))
709 for package in d.getVar("PACKAGES").split():
710 localdata = bb.data.createCopy(d)
711 pkg_name = d.getVar("PKG:%s" % package) or package
712 localdata.setVar("PKG", pkg_name)
713 localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package)
714
715 if not oe.packagedata.packaged(package, localdata):
716 continue
717
718 pkg_spdx_path = deploy_dir_spdx / "packages" / (pkg_name + ".spdx.json")
719
720 package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path)
721
722 for p in package_doc.packages:
723 if p.name == pkg_name:
724 spdx_package = p
725 break
726 else:
727 bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path))
728
729 runtime_doc = oe.spdx.SPDXDocument()
730 runtime_doc.name = "runtime-" + pkg_name
731 runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc)
732 runtime_doc.creationInfo.created = creation_time
733 runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies."
734 runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
735 runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
736 runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
737 runtime_doc.creationInfo.creators.append("Person: N/A ()")
738
739 package_ref = oe.spdx.SPDXExternalDocumentRef()
740 package_ref.externalDocumentId = "DocumentRef-package-" + package
741 package_ref.spdxDocument = package_doc.documentNamespace
742 package_ref.checksum.algorithm = "SHA1"
743 package_ref.checksum.checksumValue = package_doc_sha1
744
745 runtime_doc.externalDocumentRefs.append(package_ref)
746
747 runtime_doc.add_relationship(
748 runtime_doc.SPDXID,
749 "AMENDS",
750 "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID)
751 )
752
753 deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "")
754 seen_deps = set()
755 for dep, _ in deps.items():
756 if dep in seen_deps:
757 continue
758
759 if dep not in providers:
760 continue
761
762 dep = providers[dep]
763
764 if not oe.packagedata.packaged(dep, localdata):
765 continue
766
767 dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d)
768 dep_pkg = dep_pkg_data["PKG"]
769
770 if dep in dep_package_cache:
771 (dep_spdx_package, dep_package_ref) = dep_package_cache[dep]
772 else:
773 dep_path = deploy_dir_spdx / "packages" / ("%s.spdx.json" % dep_pkg)
774
775 spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path)
776
777 for pkg in spdx_dep_doc.packages:
778 if pkg.name == dep_pkg:
779 dep_spdx_package = pkg
780 break
781 else:
782 bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path))
783
784 dep_package_ref = oe.spdx.SPDXExternalDocumentRef()
785 dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name
786 dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace
787 dep_package_ref.checksum.algorithm = "SHA1"
788 dep_package_ref.checksum.checksumValue = spdx_dep_sha1
789
790 dep_package_cache[dep] = (dep_spdx_package, dep_package_ref)
791
792 runtime_doc.externalDocumentRefs.append(dep_package_ref)
793
794 runtime_doc.add_relationship(
795 "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID),
796 "RUNTIME_DEPENDENCY_OF",
797 "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID)
798 )
799 seen_deps.add(dep)
800
801 oe.sbom.write_doc(d, runtime_doc, "runtime", spdx_deploy, indent=get_json_indent(d))
802}
803
804addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work
805SSTATETASKS += "do_create_runtime_spdx"
806do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}"
807do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}"
808
809python do_create_runtime_spdx_setscene () {
810 sstate_setscene(d)
811}
812addtask do_create_runtime_spdx_setscene
813
814do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}"
815do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}"
816do_create_runtime_spdx[rdeptask] = "do_create_spdx"
817
818def spdx_get_src(d):
819 """
820 save patched source of the recipe in SPDX_WORKDIR.
821 """
822 import shutil
823 spdx_workdir = d.getVar('SPDXWORK')
824 spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE')
825 pn = d.getVar('PN')
826
827 workdir = d.getVar("WORKDIR")
828
829 try:
830 # The kernel class functions require it to be on work-shared, so we dont change WORKDIR
831 if not is_work_shared_spdx(d):
832 # Change the WORKDIR to make do_unpack do_patch run in another dir.
833 d.setVar('WORKDIR', spdx_workdir)
834 # Restore the original path to recipe's native sysroot (it's relative to WORKDIR).
835 d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)
836
837 # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the
838 # possibly requiring of the following tasks (such as some recipes's
839 # do_patch required 'B' existed).
840 bb.utils.mkdirhier(d.getVar('B'))
841
842 bb.build.exec_func('do_unpack', d)
843 # Copy source of kernel to spdx_workdir
844 if is_work_shared_spdx(d):
845 share_src = d.getVar('WORKDIR')
846 d.setVar('WORKDIR', spdx_workdir)
847 d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)
848 src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR')
849 bb.utils.mkdirhier(src_dir)
850 if bb.data.inherits_class('kernel',d):
851 share_src = d.getVar('STAGING_KERNEL_DIR')
852 cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/"
853 cmd_copy_shared_res = os.popen(cmd_copy_share).read()
854 bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res)
855
856 git_path = src_dir + "/.git"
857 if os.path.exists(git_path):
858 shutils.rmtree(git_path)
859
860 # Make sure gcc and kernel sources are patched only once
861 if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)):
862 bb.build.exec_func('do_patch', d)
863
864 # Some userland has no source.
865 if not os.path.exists( spdx_workdir ):
866 bb.utils.mkdirhier(spdx_workdir)
867 finally:
868 d.setVar("WORKDIR", workdir)
869
870do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
871do_rootfs[cleandirs] += "${SPDXIMAGEWORK}"
872
873ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; "
874
875do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
876do_populate_sdk[cleandirs] += "${SPDXSDKWORK}"
877POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx; "
878POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx; "
879
880python image_combine_spdx() {
881 import os
882 import oe.sbom
883 from pathlib import Path
884 from oe.rootfs import image_list_installed_packages
885
886 image_name = d.getVar("IMAGE_NAME")
887 image_link_name = d.getVar("IMAGE_LINK_NAME")
888 imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
889 img_spdxid = oe.sbom.get_image_spdxid(image_name)
890 packages = image_list_installed_packages(d)
891
892 combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages, Path(d.getVar("SPDXIMAGEWORK")))
893
894 def make_image_link(target_path, suffix):
895 if image_link_name:
896 link = imgdeploydir / (image_link_name + suffix)
897 if link != target_path:
898 link.symlink_to(os.path.relpath(target_path, link.parent))
899
900 spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.gz")
901 make_image_link(spdx_tar_path, ".spdx.tar.gz")
902}
903
904python sdk_host_combine_spdx() {
905 sdk_combine_spdx(d, "host")
906}
907
908python sdk_target_combine_spdx() {
909 sdk_combine_spdx(d, "target")
910}
911
912def sdk_combine_spdx(d, sdk_type):
913 import oe.sbom
914 from pathlib import Path
915 from oe.sdk import sdk_list_installed_packages
916
917 sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type
918 sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR"))
919 sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name)
920 sdk_packages = sdk_list_installed_packages(d, sdk_type == "target")
921 combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages, Path(d.getVar('SPDXSDKWORK')))
922
923def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages, spdx_workdir):
924 import os
925 import oe.spdx
926 import oe.sbom
927 import io
928 import json
929 from datetime import timezone, datetime
930 from pathlib import Path
931 import tarfile
932 import gzip
933
934 creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
935 deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
936 source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
937
938 doc = oe.spdx.SPDXDocument()
939 doc.name = rootfs_name
940 doc.documentNamespace = get_doc_namespace(d, doc)
941 doc.creationInfo.created = creation_time
942 doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build."
943 doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"]
944 doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass")
945 doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG"))
946 doc.creationInfo.creators.append("Person: N/A ()")
947
948 image = oe.spdx.SPDXPackage()
949 image.name = d.getVar("PN")
950 image.versionInfo = d.getVar("PV")
951 image.SPDXID = rootfs_spdxid
952 image.supplier = d.getVar("SPDX_SUPPLIER")
953
954 doc.packages.append(image)
955
956 for name in sorted(packages.keys()):
957 pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json")
958 pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path)
959
960 for p in pkg_doc.packages:
961 if p.name == name:
962 pkg_ref = oe.spdx.SPDXExternalDocumentRef()
963 pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name
964 pkg_ref.spdxDocument = pkg_doc.documentNamespace
965 pkg_ref.checksum.algorithm = "SHA1"
966 pkg_ref.checksum.checksumValue = pkg_doc_sha1
967
968 doc.externalDocumentRefs.append(pkg_ref)
969 doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID))
970 break
971 else:
972 bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path))
973
974 runtime_spdx_path = deploy_dir_spdx / "runtime" / ("runtime-" + name + ".spdx.json")
975 runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path)
976
977 runtime_ref = oe.spdx.SPDXExternalDocumentRef()
978 runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name
979 runtime_ref.spdxDocument = runtime_doc.documentNamespace
980 runtime_ref.checksum.algorithm = "SHA1"
981 runtime_ref.checksum.checksumValue = runtime_doc_sha1
982
983 # "OTHER" isn't ideal here, but I can't find a relationship that makes sense
984 doc.externalDocumentRefs.append(runtime_ref)
985 doc.add_relationship(
986 image,
987 "OTHER",
988 "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID),
989 comment="Runtime dependencies for %s" % name
990 )
991
992 image_spdx_path = spdx_workdir / (rootfs_name + ".spdx.json")
993
994 with image_spdx_path.open("wb") as f:
995 doc.to_json(f, sort_keys=True, indent=get_json_indent(d))
996
997 num_threads = int(d.getVar("BB_NUMBER_THREADS"))
998
999 visited_docs = set()
1000
1001 index = {"documents": []}
1002
1003 spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.gz")
1004 with gzip.open(spdx_tar_path, "w") as f:
1005 with tarfile.open(fileobj=f, mode="w|") as tar:
1006 def collect_spdx_document(path):
1007 nonlocal tar
1008 nonlocal deploy_dir_spdx
1009 nonlocal source_date_epoch
1010 nonlocal index
1011
1012 if path in visited_docs:
1013 return
1014
1015 visited_docs.add(path)
1016
1017 with path.open("rb") as f:
1018 doc, sha1 = oe.sbom.read_doc(f)
1019 f.seek(0)
1020
1021 if doc.documentNamespace in visited_docs:
1022 return
1023
1024 bb.note("Adding SPDX document %s" % path)
1025 visited_docs.add(doc.documentNamespace)
1026 info = tar.gettarinfo(fileobj=f)
1027
1028 info.name = doc.name + ".spdx.json"
1029 info.uid = 0
1030 info.gid = 0
1031 info.uname = "root"
1032 info.gname = "root"
1033
1034 if source_date_epoch is not None and info.mtime > int(source_date_epoch):
1035 info.mtime = int(source_date_epoch)
1036
1037 tar.addfile(info, f)
1038
1039 index["documents"].append({
1040 "filename": info.name,
1041 "documentNamespace": doc.documentNamespace,
1042 "sha1": sha1,
1043 })
1044
1045 for ref in doc.externalDocumentRefs:
1046 ref_path = deploy_dir_spdx / "by-namespace" / ref.spdxDocument.replace("/", "_")
1047 collect_spdx_document(ref_path)
1048
1049 collect_spdx_document(image_spdx_path)
1050
1051 index["documents"].sort(key=lambda x: x["filename"])
1052
1053 index_str = io.BytesIO(json.dumps(
1054 index,
1055 sort_keys=True,
1056 indent=get_json_indent(d),
1057 ).encode("utf-8"))
1058
1059 info = tarfile.TarInfo()
1060 info.name = "index.json"
1061 info.size = len(index_str.getvalue())
1062 info.uid = 0
1063 info.gid = 0
1064 info.uname = "root"
1065 info.gname = "root"
1066
1067 tar.addfile(info, fileobj=index_str)