diff options
author | Joshua Watt <JPEWhacker@gmail.com> | 2022-11-08 14:07:02 -0600 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2022-11-11 13:43:40 +0000 |
commit | 02691744256d4f267f37190e30b03b3bc3e477fc (patch) | |
tree | 2ba6eb23db297915da63f5e47d4d39ee0b39269e /meta/classes | |
parent | 25e2663a878c7cd4463652757f7c36ab5ace9b39 (diff) | |
download | poky-02691744256d4f267f37190e30b03b3bc3e477fc.tar.gz |
classes: create-spdx: Move to version specific class
In expectation of SPDX 3.0 support, move the create-spdx.bbclass ->
create-spdx-2.2.bbclass. The create-spdx.bbclass class still exists and
can be used if a user doesn't care about which specific version of SPDX
they get.
(From OE-Core rev: 9ec01fe3e59be66331c14ab5391ecb0b6f140c22)
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Alexandre Belloni <alexandre.belloni@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/classes')
-rw-r--r-- | meta/classes/create-spdx-2.2.bbclass | 1025 | ||||
-rw-r--r-- | meta/classes/create-spdx.bbclass | 1023 |
2 files changed, 1028 insertions, 1020 deletions
diff --git a/meta/classes/create-spdx-2.2.bbclass b/meta/classes/create-spdx-2.2.bbclass new file mode 100644 index 0000000000..af6afcc653 --- /dev/null +++ b/meta/classes/create-spdx-2.2.bbclass | |||
@@ -0,0 +1,1025 @@ | |||
1 | # | ||
2 | # Copyright OpenEmbedded Contributors | ||
3 | # | ||
4 | # SPDX-License-Identifier: GPL-2.0-only | ||
5 | # | ||
6 | |||
7 | DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${MACHINE}" | ||
8 | |||
9 | # The product name that the CVE database uses. Defaults to BPN, but may need to | ||
10 | # be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). | ||
11 | CVE_PRODUCT ??= "${BPN}" | ||
12 | CVE_VERSION ??= "${PV}" | ||
13 | |||
14 | SPDXDIR ??= "${WORKDIR}/spdx" | ||
15 | SPDXDEPLOY = "${SPDXDIR}/deploy" | ||
16 | SPDXWORK = "${SPDXDIR}/work" | ||
17 | |||
18 | SPDX_TOOL_NAME ??= "oe-spdx-creator" | ||
19 | SPDX_TOOL_VERSION ??= "1.0" | ||
20 | |||
21 | SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" | ||
22 | |||
23 | SPDX_INCLUDE_SOURCES ??= "0" | ||
24 | SPDX_ARCHIVE_SOURCES ??= "0" | ||
25 | SPDX_ARCHIVE_PACKAGED ??= "0" | ||
26 | |||
27 | SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" | ||
28 | SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc" | ||
29 | SPDX_PRETTY ??= "0" | ||
30 | |||
31 | SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" | ||
32 | |||
33 | SPDX_ORG ??= "OpenEmbedded ()" | ||
34 | SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" | ||
35 | SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ | ||
36 | this recipe. For SPDX documents create using this class during the build, this \ | ||
37 | is the contact information for the person or organization who is doing the \ | ||
38 | build." | ||
39 | |||
40 | def extract_licenses(filename): | ||
41 | import re | ||
42 | |||
43 | lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE) | ||
44 | |||
45 | try: | ||
46 | with open(filename, 'rb') as f: | ||
47 | size = min(15000, os.stat(filename).st_size) | ||
48 | txt = f.read(size) | ||
49 | licenses = re.findall(lic_regex, txt) | ||
50 | if licenses: | ||
51 | ascii_licenses = [lic.decode('ascii') for lic in licenses] | ||
52 | return ascii_licenses | ||
53 | except Exception as e: | ||
54 | bb.warn(f"Exception reading {filename}: {e}") | ||
55 | return None | ||
56 | |||
57 | def get_doc_namespace(d, doc): | ||
58 | import uuid | ||
59 | namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) | ||
60 | return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) | ||
61 | |||
62 | def create_annotation(d, comment): | ||
63 | from datetime import datetime, timezone | ||
64 | |||
65 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | ||
66 | annotation = oe.spdx.SPDXAnnotation() | ||
67 | annotation.annotationDate = creation_time | ||
68 | annotation.annotationType = "OTHER" | ||
69 | annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) | ||
70 | annotation.comment = comment | ||
71 | return annotation | ||
72 | |||
73 | def recipe_spdx_is_native(d, recipe): | ||
74 | return any(a.annotationType == "OTHER" and | ||
75 | a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and | ||
76 | a.comment == "isNative" for a in recipe.annotations) | ||
77 | |||
78 | def is_work_shared_spdx(d): | ||
79 | return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR')) | ||
80 | |||
81 | def get_json_indent(d): | ||
82 | if d.getVar("SPDX_PRETTY") == "1": | ||
83 | return 2 | ||
84 | return None | ||
85 | |||
86 | python() { | ||
87 | import json | ||
88 | if d.getVar("SPDX_LICENSE_DATA"): | ||
89 | return | ||
90 | |||
91 | with open(d.getVar("SPDX_LICENSES"), "r") as f: | ||
92 | data = json.load(f) | ||
93 | # Transform the license array to a dictionary | ||
94 | data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} | ||
95 | d.setVar("SPDX_LICENSE_DATA", data) | ||
96 | } | ||
97 | |||
98 | def convert_license_to_spdx(lic, document, d, existing={}): | ||
99 | from pathlib import Path | ||
100 | import oe.spdx | ||
101 | |||
102 | license_data = d.getVar("SPDX_LICENSE_DATA") | ||
103 | extracted = {} | ||
104 | |||
105 | def add_extracted_license(ident, name): | ||
106 | nonlocal document | ||
107 | |||
108 | if name in extracted: | ||
109 | return | ||
110 | |||
111 | extracted_info = oe.spdx.SPDXExtractedLicensingInfo() | ||
112 | extracted_info.name = name | ||
113 | extracted_info.licenseId = ident | ||
114 | extracted_info.extractedText = None | ||
115 | |||
116 | if name == "PD": | ||
117 | # Special-case this. | ||
118 | extracted_info.extractedText = "Software released to the public domain" | ||
119 | else: | ||
120 | # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH | ||
121 | for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): | ||
122 | try: | ||
123 | with (Path(directory) / name).open(errors="replace") as f: | ||
124 | extracted_info.extractedText = f.read() | ||
125 | break | ||
126 | except FileNotFoundError: | ||
127 | pass | ||
128 | if extracted_info.extractedText is None: | ||
129 | # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set | ||
130 | filename = d.getVarFlag('NO_GENERIC_LICENSE', name) | ||
131 | if filename: | ||
132 | filename = d.expand("${S}/" + filename) | ||
133 | with open(filename, errors="replace") as f: | ||
134 | extracted_info.extractedText = f.read() | ||
135 | else: | ||
136 | bb.error("Cannot find any text for license %s" % name) | ||
137 | |||
138 | extracted[name] = extracted_info | ||
139 | document.hasExtractedLicensingInfos.append(extracted_info) | ||
140 | |||
141 | def convert(l): | ||
142 | if l == "(" or l == ")": | ||
143 | return l | ||
144 | |||
145 | if l == "&": | ||
146 | return "AND" | ||
147 | |||
148 | if l == "|": | ||
149 | return "OR" | ||
150 | |||
151 | if l == "CLOSED": | ||
152 | return "NONE" | ||
153 | |||
154 | spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l | ||
155 | if spdx_license in license_data["licenses"]: | ||
156 | return spdx_license | ||
157 | |||
158 | try: | ||
159 | spdx_license = existing[l] | ||
160 | except KeyError: | ||
161 | spdx_license = "LicenseRef-" + l | ||
162 | add_extracted_license(spdx_license, l) | ||
163 | |||
164 | return spdx_license | ||
165 | |||
166 | lic_split = lic.replace("(", " ( ").replace(")", " ) ").split() | ||
167 | |||
168 | return ' '.join(convert(l) for l in lic_split) | ||
169 | |||
170 | def process_sources(d): | ||
171 | pn = d.getVar('PN') | ||
172 | assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() | ||
173 | if pn in assume_provided: | ||
174 | for p in d.getVar("PROVIDES").split(): | ||
175 | if p != pn: | ||
176 | pn = p | ||
177 | break | ||
178 | |||
179 | # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted, | ||
180 | # so avoid archiving source here. | ||
181 | if pn.startswith('glibc-locale'): | ||
182 | return False | ||
183 | if d.getVar('PN') == "libtool-cross": | ||
184 | return False | ||
185 | if d.getVar('PN') == "libgcc-initial": | ||
186 | return False | ||
187 | if d.getVar('PN') == "shadow-sysroot": | ||
188 | return False | ||
189 | |||
190 | # We just archive gcc-source for all the gcc related recipes | ||
191 | if d.getVar('BPN') in ['gcc', 'libgcc']: | ||
192 | bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) | ||
193 | return False | ||
194 | |||
195 | return True | ||
196 | |||
197 | |||
198 | def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): | ||
199 | from pathlib import Path | ||
200 | import oe.spdx | ||
201 | import hashlib | ||
202 | |||
203 | source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") | ||
204 | if source_date_epoch: | ||
205 | source_date_epoch = int(source_date_epoch) | ||
206 | |||
207 | sha1s = [] | ||
208 | spdx_files = [] | ||
209 | |||
210 | file_counter = 1 | ||
211 | for subdir, dirs, files in os.walk(topdir): | ||
212 | dirs[:] = [d for d in dirs if d not in ignore_dirs] | ||
213 | if subdir == str(topdir): | ||
214 | dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] | ||
215 | |||
216 | for file in files: | ||
217 | filepath = Path(subdir) / file | ||
218 | filename = str(filepath.relative_to(topdir)) | ||
219 | |||
220 | if not filepath.is_symlink() and filepath.is_file(): | ||
221 | spdx_file = oe.spdx.SPDXFile() | ||
222 | spdx_file.SPDXID = get_spdxid(file_counter) | ||
223 | for t in get_types(filepath): | ||
224 | spdx_file.fileTypes.append(t) | ||
225 | spdx_file.fileName = filename | ||
226 | |||
227 | if archive is not None: | ||
228 | with filepath.open("rb") as f: | ||
229 | info = archive.gettarinfo(fileobj=f) | ||
230 | info.name = filename | ||
231 | info.uid = 0 | ||
232 | info.gid = 0 | ||
233 | info.uname = "root" | ||
234 | info.gname = "root" | ||
235 | |||
236 | if source_date_epoch is not None and info.mtime > source_date_epoch: | ||
237 | info.mtime = source_date_epoch | ||
238 | |||
239 | archive.addfile(info, f) | ||
240 | |||
241 | sha1 = bb.utils.sha1_file(filepath) | ||
242 | sha1s.append(sha1) | ||
243 | spdx_file.checksums.append(oe.spdx.SPDXChecksum( | ||
244 | algorithm="SHA1", | ||
245 | checksumValue=sha1, | ||
246 | )) | ||
247 | spdx_file.checksums.append(oe.spdx.SPDXChecksum( | ||
248 | algorithm="SHA256", | ||
249 | checksumValue=bb.utils.sha256_file(filepath), | ||
250 | )) | ||
251 | |||
252 | if "SOURCE" in spdx_file.fileTypes: | ||
253 | extracted_lics = extract_licenses(filepath) | ||
254 | if extracted_lics: | ||
255 | spdx_file.licenseInfoInFiles = extracted_lics | ||
256 | |||
257 | doc.files.append(spdx_file) | ||
258 | doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) | ||
259 | spdx_pkg.hasFiles.append(spdx_file.SPDXID) | ||
260 | |||
261 | spdx_files.append(spdx_file) | ||
262 | |||
263 | file_counter += 1 | ||
264 | |||
265 | sha1s.sort() | ||
266 | verifier = hashlib.sha1() | ||
267 | for v in sha1s: | ||
268 | verifier.update(v.encode("utf-8")) | ||
269 | spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest() | ||
270 | |||
271 | return spdx_files | ||
272 | |||
273 | |||
274 | def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources): | ||
275 | from pathlib import Path | ||
276 | import hashlib | ||
277 | import oe.packagedata | ||
278 | import oe.spdx | ||
279 | |||
280 | debug_search_paths = [ | ||
281 | Path(d.getVar('PKGD')), | ||
282 | Path(d.getVar('STAGING_DIR_TARGET')), | ||
283 | Path(d.getVar('STAGING_DIR_NATIVE')), | ||
284 | Path(d.getVar('STAGING_KERNEL_DIR')), | ||
285 | ] | ||
286 | |||
287 | pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) | ||
288 | |||
289 | if pkg_data is None: | ||
290 | return | ||
291 | |||
292 | for file_path, file_data in pkg_data["files_info"].items(): | ||
293 | if not "debugsrc" in file_data: | ||
294 | continue | ||
295 | |||
296 | for pkg_file in package_files: | ||
297 | if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): | ||
298 | break | ||
299 | else: | ||
300 | bb.fatal("No package file found for %s" % str(file_path)) | ||
301 | continue | ||
302 | |||
303 | for debugsrc in file_data["debugsrc"]: | ||
304 | ref_id = "NOASSERTION" | ||
305 | for search in debug_search_paths: | ||
306 | if debugsrc.startswith("/usr/src/kernel"): | ||
307 | debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') | ||
308 | else: | ||
309 | debugsrc_path = search / debugsrc.lstrip("/") | ||
310 | if not debugsrc_path.exists(): | ||
311 | continue | ||
312 | |||
313 | file_sha256 = bb.utils.sha256_file(debugsrc_path) | ||
314 | |||
315 | if file_sha256 in sources: | ||
316 | source_file = sources[file_sha256] | ||
317 | |||
318 | doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace) | ||
319 | if doc_ref is None: | ||
320 | doc_ref = oe.spdx.SPDXExternalDocumentRef() | ||
321 | doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name | ||
322 | doc_ref.spdxDocument = source_file.doc.documentNamespace | ||
323 | doc_ref.checksum.algorithm = "SHA1" | ||
324 | doc_ref.checksum.checksumValue = source_file.doc_sha1 | ||
325 | package_doc.externalDocumentRefs.append(doc_ref) | ||
326 | |||
327 | ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID) | ||
328 | else: | ||
329 | bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) | ||
330 | break | ||
331 | else: | ||
332 | bb.debug(1, "Debug source %s not found" % debugsrc) | ||
333 | |||
334 | package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc) | ||
335 | |||
336 | def collect_dep_recipes(d, doc, spdx_recipe): | ||
337 | from pathlib import Path | ||
338 | import oe.sbom | ||
339 | import oe.spdx | ||
340 | |||
341 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) | ||
342 | |||
343 | dep_recipes = [] | ||
344 | taskdepdata = d.getVar("BB_TASKDEPDATA", False) | ||
345 | deps = sorted(set( | ||
346 | dep[0] for dep in taskdepdata.values() if | ||
347 | dep[1] == "do_create_spdx" and dep[0] != d.getVar("PN") | ||
348 | )) | ||
349 | for dep_pn in deps: | ||
350 | dep_recipe_path = deploy_dir_spdx / "recipes" / ("recipe-%s.spdx.json" % dep_pn) | ||
351 | |||
352 | spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) | ||
353 | |||
354 | for pkg in spdx_dep_doc.packages: | ||
355 | if pkg.name == dep_pn: | ||
356 | spdx_dep_recipe = pkg | ||
357 | break | ||
358 | else: | ||
359 | continue | ||
360 | |||
361 | dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe)) | ||
362 | |||
363 | dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() | ||
364 | dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name | ||
365 | dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace | ||
366 | dep_recipe_ref.checksum.algorithm = "SHA1" | ||
367 | dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 | ||
368 | |||
369 | doc.externalDocumentRefs.append(dep_recipe_ref) | ||
370 | |||
371 | doc.add_relationship( | ||
372 | "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID), | ||
373 | "BUILD_DEPENDENCY_OF", | ||
374 | spdx_recipe | ||
375 | ) | ||
376 | |||
377 | return dep_recipes | ||
378 | |||
379 | collect_dep_recipes[vardepsexclude] += "BB_TASKDEPDATA" | ||
380 | |||
381 | |||
382 | def collect_dep_sources(d, dep_recipes): | ||
383 | import oe.sbom | ||
384 | |||
385 | sources = {} | ||
386 | for dep in dep_recipes: | ||
387 | # Don't collect sources from native recipes as they | ||
388 | # match non-native sources also. | ||
389 | if recipe_spdx_is_native(d, dep.recipe): | ||
390 | continue | ||
391 | recipe_files = set(dep.recipe.hasFiles) | ||
392 | |||
393 | for spdx_file in dep.doc.files: | ||
394 | if spdx_file.SPDXID not in recipe_files: | ||
395 | continue | ||
396 | |||
397 | if "SOURCE" in spdx_file.fileTypes: | ||
398 | for checksum in spdx_file.checksums: | ||
399 | if checksum.algorithm == "SHA256": | ||
400 | sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) | ||
401 | break | ||
402 | |||
403 | return sources | ||
404 | |||
405 | |||
406 | python do_create_spdx() { | ||
407 | from datetime import datetime, timezone | ||
408 | import oe.sbom | ||
409 | import oe.spdx | ||
410 | import uuid | ||
411 | from pathlib import Path | ||
412 | from contextlib import contextmanager | ||
413 | import oe.cve_check | ||
414 | |||
415 | @contextmanager | ||
416 | def optional_tarfile(name, guard, mode="w"): | ||
417 | import tarfile | ||
418 | import bb.compress.zstd | ||
419 | |||
420 | num_threads = int(d.getVar("BB_NUMBER_THREADS")) | ||
421 | |||
422 | if guard: | ||
423 | name.parent.mkdir(parents=True, exist_ok=True) | ||
424 | with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f: | ||
425 | with tarfile.open(fileobj=f, mode=mode + "|") as tf: | ||
426 | yield tf | ||
427 | else: | ||
428 | yield None | ||
429 | |||
430 | |||
431 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) | ||
432 | spdx_workdir = Path(d.getVar("SPDXWORK")) | ||
433 | include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" | ||
434 | archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" | ||
435 | archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" | ||
436 | |||
437 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | ||
438 | |||
439 | doc = oe.spdx.SPDXDocument() | ||
440 | |||
441 | doc.name = "recipe-" + d.getVar("PN") | ||
442 | doc.documentNamespace = get_doc_namespace(d, doc) | ||
443 | doc.creationInfo.created = creation_time | ||
444 | doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." | ||
445 | doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] | ||
446 | doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") | ||
447 | doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) | ||
448 | doc.creationInfo.creators.append("Person: N/A ()") | ||
449 | |||
450 | recipe = oe.spdx.SPDXPackage() | ||
451 | recipe.name = d.getVar("PN") | ||
452 | recipe.versionInfo = d.getVar("PV") | ||
453 | recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) | ||
454 | recipe.supplier = d.getVar("SPDX_SUPPLIER") | ||
455 | if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): | ||
456 | recipe.annotations.append(create_annotation(d, "isNative")) | ||
457 | |||
458 | for s in d.getVar('SRC_URI').split(): | ||
459 | if not s.startswith("file://"): | ||
460 | s = s.split(';')[0] | ||
461 | recipe.downloadLocation = s | ||
462 | break | ||
463 | else: | ||
464 | recipe.downloadLocation = "NOASSERTION" | ||
465 | |||
466 | homepage = d.getVar("HOMEPAGE") | ||
467 | if homepage: | ||
468 | recipe.homepage = homepage | ||
469 | |||
470 | license = d.getVar("LICENSE") | ||
471 | if license: | ||
472 | recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) | ||
473 | |||
474 | summary = d.getVar("SUMMARY") | ||
475 | if summary: | ||
476 | recipe.summary = summary | ||
477 | |||
478 | description = d.getVar("DESCRIPTION") | ||
479 | if description: | ||
480 | recipe.description = description | ||
481 | |||
482 | # Some CVEs may be patched during the build process without incrementing the version number, | ||
483 | # so querying for CVEs based on the CPE id can lead to false positives. To account for this, | ||
484 | # save the CVEs fixed by patches to source information field in the SPDX. | ||
485 | patched_cves = oe.cve_check.get_patched_cves(d) | ||
486 | patched_cves = list(patched_cves) | ||
487 | patched_cves = ' '.join(patched_cves) | ||
488 | if patched_cves: | ||
489 | recipe.sourceInfo = "CVEs fixed: " + patched_cves | ||
490 | |||
491 | cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) | ||
492 | if cpe_ids: | ||
493 | for cpe_id in cpe_ids: | ||
494 | cpe = oe.spdx.SPDXExternalReference() | ||
495 | cpe.referenceCategory = "SECURITY" | ||
496 | cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type" | ||
497 | cpe.referenceLocator = cpe_id | ||
498 | recipe.externalRefs.append(cpe) | ||
499 | |||
500 | doc.packages.append(recipe) | ||
501 | doc.add_relationship(doc, "DESCRIBES", recipe) | ||
502 | |||
503 | if process_sources(d) and include_sources: | ||
504 | recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") | ||
505 | with optional_tarfile(recipe_archive, archive_sources) as archive: | ||
506 | spdx_get_src(d) | ||
507 | |||
508 | add_package_files( | ||
509 | d, | ||
510 | doc, | ||
511 | recipe, | ||
512 | spdx_workdir, | ||
513 | lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter), | ||
514 | lambda filepath: ["SOURCE"], | ||
515 | ignore_dirs=[".git"], | ||
516 | ignore_top_level_dirs=["temp"], | ||
517 | archive=archive, | ||
518 | ) | ||
519 | |||
520 | if archive is not None: | ||
521 | recipe.packageFileName = str(recipe_archive.name) | ||
522 | |||
523 | dep_recipes = collect_dep_recipes(d, doc, recipe) | ||
524 | |||
525 | doc_sha1 = oe.sbom.write_doc(d, doc, "recipes", indent=get_json_indent(d)) | ||
526 | dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) | ||
527 | |||
528 | recipe_ref = oe.spdx.SPDXExternalDocumentRef() | ||
529 | recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name | ||
530 | recipe_ref.spdxDocument = doc.documentNamespace | ||
531 | recipe_ref.checksum.algorithm = "SHA1" | ||
532 | recipe_ref.checksum.checksumValue = doc_sha1 | ||
533 | |||
534 | sources = collect_dep_sources(d, dep_recipes) | ||
535 | found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} | ||
536 | |||
537 | if not recipe_spdx_is_native(d, recipe): | ||
538 | bb.build.exec_func("read_subpackage_metadata", d) | ||
539 | |||
540 | pkgdest = Path(d.getVar("PKGDEST")) | ||
541 | for package in d.getVar("PACKAGES").split(): | ||
542 | if not oe.packagedata.packaged(package, d): | ||
543 | continue | ||
544 | |||
545 | package_doc = oe.spdx.SPDXDocument() | ||
546 | pkg_name = d.getVar("PKG:%s" % package) or package | ||
547 | package_doc.name = pkg_name | ||
548 | package_doc.documentNamespace = get_doc_namespace(d, package_doc) | ||
549 | package_doc.creationInfo.created = creation_time | ||
550 | package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." | ||
551 | package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] | ||
552 | package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") | ||
553 | package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) | ||
554 | package_doc.creationInfo.creators.append("Person: N/A ()") | ||
555 | package_doc.externalDocumentRefs.append(recipe_ref) | ||
556 | |||
557 | package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE") | ||
558 | |||
559 | spdx_package = oe.spdx.SPDXPackage() | ||
560 | |||
561 | spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) | ||
562 | spdx_package.name = pkg_name | ||
563 | spdx_package.versionInfo = d.getVar("PV") | ||
564 | spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) | ||
565 | spdx_package.supplier = d.getVar("SPDX_SUPPLIER") | ||
566 | |||
567 | package_doc.packages.append(spdx_package) | ||
568 | |||
569 | package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) | ||
570 | package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) | ||
571 | |||
572 | package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst") | ||
573 | with optional_tarfile(package_archive, archive_packaged) as archive: | ||
574 | package_files = add_package_files( | ||
575 | d, | ||
576 | package_doc, | ||
577 | spdx_package, | ||
578 | pkgdest / package, | ||
579 | lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), | ||
580 | lambda filepath: ["BINARY"], | ||
581 | ignore_top_level_dirs=['CONTROL', 'DEBIAN'], | ||
582 | archive=archive, | ||
583 | ) | ||
584 | |||
585 | if archive is not None: | ||
586 | spdx_package.packageFileName = str(package_archive.name) | ||
587 | |||
588 | add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources) | ||
589 | |||
590 | oe.sbom.write_doc(d, package_doc, "packages", indent=get_json_indent(d)) | ||
591 | } | ||
592 | # NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source | ||
593 | addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work | ||
594 | |||
595 | SSTATETASKS += "do_create_spdx" | ||
596 | do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" | ||
597 | do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" | ||
598 | |||
599 | python do_create_spdx_setscene () { | ||
600 | sstate_setscene(d) | ||
601 | } | ||
602 | addtask do_create_spdx_setscene | ||
603 | |||
604 | do_create_spdx[dirs] = "${SPDXWORK}" | ||
605 | do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" | ||
606 | do_create_spdx[depends] += "${PATCHDEPENDENCY}" | ||
607 | do_create_spdx[deptask] = "do_create_spdx" | ||
608 | |||
609 | def collect_package_providers(d): | ||
610 | from pathlib import Path | ||
611 | import oe.sbom | ||
612 | import oe.spdx | ||
613 | import json | ||
614 | |||
615 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) | ||
616 | |||
617 | providers = {} | ||
618 | |||
619 | taskdepdata = d.getVar("BB_TASKDEPDATA", False) | ||
620 | deps = sorted(set( | ||
621 | dep[0] for dep in taskdepdata.values() if dep[0] != d.getVar("PN") | ||
622 | )) | ||
623 | deps.append(d.getVar("PN")) | ||
624 | |||
625 | for dep_pn in deps: | ||
626 | recipe_data = oe.packagedata.read_pkgdata(dep_pn, d) | ||
627 | |||
628 | for pkg in recipe_data.get("PACKAGES", "").split(): | ||
629 | |||
630 | pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, d) | ||
631 | rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items()) | ||
632 | rprovides.add(pkg) | ||
633 | |||
634 | for r in rprovides: | ||
635 | providers[r] = pkg | ||
636 | |||
637 | return providers | ||
638 | |||
639 | collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" | ||
640 | |||
641 | python do_create_runtime_spdx() { | ||
642 | from datetime import datetime, timezone | ||
643 | import oe.sbom | ||
644 | import oe.spdx | ||
645 | import oe.packagedata | ||
646 | from pathlib import Path | ||
647 | |||
648 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) | ||
649 | spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) | ||
650 | is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) | ||
651 | |||
652 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | ||
653 | |||
654 | providers = collect_package_providers(d) | ||
655 | |||
656 | if not is_native: | ||
657 | bb.build.exec_func("read_subpackage_metadata", d) | ||
658 | |||
659 | dep_package_cache = {} | ||
660 | |||
661 | pkgdest = Path(d.getVar("PKGDEST")) | ||
662 | for package in d.getVar("PACKAGES").split(): | ||
663 | localdata = bb.data.createCopy(d) | ||
664 | pkg_name = d.getVar("PKG:%s" % package) or package | ||
665 | localdata.setVar("PKG", pkg_name) | ||
666 | localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) | ||
667 | |||
668 | if not oe.packagedata.packaged(package, localdata): | ||
669 | continue | ||
670 | |||
671 | pkg_spdx_path = deploy_dir_spdx / "packages" / (pkg_name + ".spdx.json") | ||
672 | |||
673 | package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) | ||
674 | |||
675 | for p in package_doc.packages: | ||
676 | if p.name == pkg_name: | ||
677 | spdx_package = p | ||
678 | break | ||
679 | else: | ||
680 | bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path)) | ||
681 | |||
682 | runtime_doc = oe.spdx.SPDXDocument() | ||
683 | runtime_doc.name = "runtime-" + pkg_name | ||
684 | runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) | ||
685 | runtime_doc.creationInfo.created = creation_time | ||
686 | runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." | ||
687 | runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] | ||
688 | runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") | ||
689 | runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) | ||
690 | runtime_doc.creationInfo.creators.append("Person: N/A ()") | ||
691 | |||
692 | package_ref = oe.spdx.SPDXExternalDocumentRef() | ||
693 | package_ref.externalDocumentId = "DocumentRef-package-" + package | ||
694 | package_ref.spdxDocument = package_doc.documentNamespace | ||
695 | package_ref.checksum.algorithm = "SHA1" | ||
696 | package_ref.checksum.checksumValue = package_doc_sha1 | ||
697 | |||
698 | runtime_doc.externalDocumentRefs.append(package_ref) | ||
699 | |||
700 | runtime_doc.add_relationship( | ||
701 | runtime_doc.SPDXID, | ||
702 | "AMENDS", | ||
703 | "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID) | ||
704 | ) | ||
705 | |||
706 | deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") | ||
707 | seen_deps = set() | ||
708 | for dep, _ in deps.items(): | ||
709 | if dep in seen_deps: | ||
710 | continue | ||
711 | |||
712 | if dep not in providers: | ||
713 | continue | ||
714 | |||
715 | dep = providers[dep] | ||
716 | |||
717 | if not oe.packagedata.packaged(dep, localdata): | ||
718 | continue | ||
719 | |||
720 | dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) | ||
721 | dep_pkg = dep_pkg_data["PKG"] | ||
722 | |||
723 | if dep in dep_package_cache: | ||
724 | (dep_spdx_package, dep_package_ref) = dep_package_cache[dep] | ||
725 | else: | ||
726 | dep_path = deploy_dir_spdx / "packages" / ("%s.spdx.json" % dep_pkg) | ||
727 | |||
728 | spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) | ||
729 | |||
730 | for pkg in spdx_dep_doc.packages: | ||
731 | if pkg.name == dep_pkg: | ||
732 | dep_spdx_package = pkg | ||
733 | break | ||
734 | else: | ||
735 | bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path)) | ||
736 | |||
737 | dep_package_ref = oe.spdx.SPDXExternalDocumentRef() | ||
738 | dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name | ||
739 | dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace | ||
740 | dep_package_ref.checksum.algorithm = "SHA1" | ||
741 | dep_package_ref.checksum.checksumValue = spdx_dep_sha1 | ||
742 | |||
743 | dep_package_cache[dep] = (dep_spdx_package, dep_package_ref) | ||
744 | |||
745 | runtime_doc.externalDocumentRefs.append(dep_package_ref) | ||
746 | |||
747 | runtime_doc.add_relationship( | ||
748 | "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID), | ||
749 | "RUNTIME_DEPENDENCY_OF", | ||
750 | "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID) | ||
751 | ) | ||
752 | seen_deps.add(dep) | ||
753 | |||
754 | oe.sbom.write_doc(d, runtime_doc, "runtime", spdx_deploy, indent=get_json_indent(d)) | ||
755 | } | ||
756 | |||
757 | addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work | ||
758 | SSTATETASKS += "do_create_runtime_spdx" | ||
759 | do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" | ||
760 | do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" | ||
761 | |||
762 | python do_create_runtime_spdx_setscene () { | ||
763 | sstate_setscene(d) | ||
764 | } | ||
765 | addtask do_create_runtime_spdx_setscene | ||
766 | |||
767 | do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" | ||
768 | do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" | ||
769 | do_create_runtime_spdx[rdeptask] = "do_create_spdx" | ||
770 | |||
771 | def spdx_get_src(d): | ||
772 | """ | ||
773 | save patched source of the recipe in SPDX_WORKDIR. | ||
774 | """ | ||
775 | import shutil | ||
776 | spdx_workdir = d.getVar('SPDXWORK') | ||
777 | spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') | ||
778 | pn = d.getVar('PN') | ||
779 | |||
780 | workdir = d.getVar("WORKDIR") | ||
781 | |||
782 | try: | ||
783 | # The kernel class functions require it to be on work-shared, so we dont change WORKDIR | ||
784 | if not is_work_shared_spdx(d): | ||
785 | # Change the WORKDIR to make do_unpack do_patch run in another dir. | ||
786 | d.setVar('WORKDIR', spdx_workdir) | ||
787 | # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). | ||
788 | d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) | ||
789 | |||
790 | # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the | ||
791 | # possibly requiring of the following tasks (such as some recipes's | ||
792 | # do_patch required 'B' existed). | ||
793 | bb.utils.mkdirhier(d.getVar('B')) | ||
794 | |||
795 | bb.build.exec_func('do_unpack', d) | ||
796 | # Copy source of kernel to spdx_workdir | ||
797 | if is_work_shared_spdx(d): | ||
798 | d.setVar('WORKDIR', spdx_workdir) | ||
799 | d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) | ||
800 | src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR') | ||
801 | bb.utils.mkdirhier(src_dir) | ||
802 | if bb.data.inherits_class('kernel',d): | ||
803 | share_src = d.getVar('STAGING_KERNEL_DIR') | ||
804 | cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/" | ||
805 | cmd_copy_kernel_result = os.popen(cmd_copy_share).read() | ||
806 | bb.note("cmd_copy_kernel_result = " + cmd_copy_kernel_result) | ||
807 | |||
808 | git_path = src_dir + "/.git" | ||
809 | if os.path.exists(git_path): | ||
810 | shutils.rmtree(git_path) | ||
811 | |||
812 | # Make sure gcc and kernel sources are patched only once | ||
813 | if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)): | ||
814 | bb.build.exec_func('do_patch', d) | ||
815 | |||
816 | # Some userland has no source. | ||
817 | if not os.path.exists( spdx_workdir ): | ||
818 | bb.utils.mkdirhier(spdx_workdir) | ||
819 | finally: | ||
820 | d.setVar("WORKDIR", workdir) | ||
821 | |||
822 | do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" | ||
823 | |||
824 | ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; " | ||
825 | |||
826 | do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" | ||
827 | POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx; " | ||
828 | POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx; " | ||
829 | |||
830 | python image_combine_spdx() { | ||
831 | import os | ||
832 | import oe.sbom | ||
833 | from pathlib import Path | ||
834 | from oe.rootfs import image_list_installed_packages | ||
835 | |||
836 | image_name = d.getVar("IMAGE_NAME") | ||
837 | image_link_name = d.getVar("IMAGE_LINK_NAME") | ||
838 | imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) | ||
839 | img_spdxid = oe.sbom.get_image_spdxid(image_name) | ||
840 | packages = image_list_installed_packages(d) | ||
841 | |||
842 | combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages) | ||
843 | |||
844 | def make_image_link(target_path, suffix): | ||
845 | if image_link_name: | ||
846 | link = imgdeploydir / (image_link_name + suffix) | ||
847 | if link != target_path: | ||
848 | link.symlink_to(os.path.relpath(target_path, link.parent)) | ||
849 | |||
850 | image_spdx_path = imgdeploydir / (image_name + ".spdx.json") | ||
851 | make_image_link(image_spdx_path, ".spdx.json") | ||
852 | spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") | ||
853 | make_image_link(spdx_tar_path, ".spdx.tar.zst") | ||
854 | spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json") | ||
855 | make_image_link(spdx_index_path, ".spdx.index.json") | ||
856 | } | ||
857 | |||
858 | python sdk_host_combine_spdx() { | ||
859 | sdk_combine_spdx(d, "host") | ||
860 | } | ||
861 | |||
862 | python sdk_target_combine_spdx() { | ||
863 | sdk_combine_spdx(d, "target") | ||
864 | } | ||
865 | |||
866 | def sdk_combine_spdx(d, sdk_type): | ||
867 | import oe.sbom | ||
868 | from pathlib import Path | ||
869 | from oe.sdk import sdk_list_installed_packages | ||
870 | |||
871 | sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type | ||
872 | sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) | ||
873 | sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name) | ||
874 | sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") | ||
875 | combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages) | ||
876 | |||
877 | def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages): | ||
878 | import os | ||
879 | import oe.spdx | ||
880 | import oe.sbom | ||
881 | import io | ||
882 | import json | ||
883 | from datetime import timezone, datetime | ||
884 | from pathlib import Path | ||
885 | import tarfile | ||
886 | import bb.compress.zstd | ||
887 | |||
888 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | ||
889 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) | ||
890 | source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") | ||
891 | |||
892 | doc = oe.spdx.SPDXDocument() | ||
893 | doc.name = rootfs_name | ||
894 | doc.documentNamespace = get_doc_namespace(d, doc) | ||
895 | doc.creationInfo.created = creation_time | ||
896 | doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." | ||
897 | doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] | ||
898 | doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") | ||
899 | doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) | ||
900 | doc.creationInfo.creators.append("Person: N/A ()") | ||
901 | |||
902 | image = oe.spdx.SPDXPackage() | ||
903 | image.name = d.getVar("PN") | ||
904 | image.versionInfo = d.getVar("PV") | ||
905 | image.SPDXID = rootfs_spdxid | ||
906 | image.supplier = d.getVar("SPDX_SUPPLIER") | ||
907 | |||
908 | doc.packages.append(image) | ||
909 | |||
910 | for name in sorted(packages.keys()): | ||
911 | pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json") | ||
912 | pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) | ||
913 | |||
914 | for p in pkg_doc.packages: | ||
915 | if p.name == name: | ||
916 | pkg_ref = oe.spdx.SPDXExternalDocumentRef() | ||
917 | pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name | ||
918 | pkg_ref.spdxDocument = pkg_doc.documentNamespace | ||
919 | pkg_ref.checksum.algorithm = "SHA1" | ||
920 | pkg_ref.checksum.checksumValue = pkg_doc_sha1 | ||
921 | |||
922 | doc.externalDocumentRefs.append(pkg_ref) | ||
923 | doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) | ||
924 | break | ||
925 | else: | ||
926 | bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) | ||
927 | |||
928 | runtime_spdx_path = deploy_dir_spdx / "runtime" / ("runtime-" + name + ".spdx.json") | ||
929 | runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) | ||
930 | |||
931 | runtime_ref = oe.spdx.SPDXExternalDocumentRef() | ||
932 | runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name | ||
933 | runtime_ref.spdxDocument = runtime_doc.documentNamespace | ||
934 | runtime_ref.checksum.algorithm = "SHA1" | ||
935 | runtime_ref.checksum.checksumValue = runtime_doc_sha1 | ||
936 | |||
937 | # "OTHER" isn't ideal here, but I can't find a relationship that makes sense | ||
938 | doc.externalDocumentRefs.append(runtime_ref) | ||
939 | doc.add_relationship( | ||
940 | image, | ||
941 | "OTHER", | ||
942 | "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), | ||
943 | comment="Runtime dependencies for %s" % name | ||
944 | ) | ||
945 | |||
946 | image_spdx_path = rootfs_deploydir / (rootfs_name + ".spdx.json") | ||
947 | |||
948 | with image_spdx_path.open("wb") as f: | ||
949 | doc.to_json(f, sort_keys=True, indent=get_json_indent(d)) | ||
950 | |||
951 | num_threads = int(d.getVar("BB_NUMBER_THREADS")) | ||
952 | |||
953 | visited_docs = set() | ||
954 | |||
955 | index = {"documents": []} | ||
956 | |||
957 | spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") | ||
958 | with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f: | ||
959 | with tarfile.open(fileobj=f, mode="w|") as tar: | ||
960 | def collect_spdx_document(path): | ||
961 | nonlocal tar | ||
962 | nonlocal deploy_dir_spdx | ||
963 | nonlocal source_date_epoch | ||
964 | nonlocal index | ||
965 | |||
966 | if path in visited_docs: | ||
967 | return | ||
968 | |||
969 | visited_docs.add(path) | ||
970 | |||
971 | with path.open("rb") as f: | ||
972 | doc, sha1 = oe.sbom.read_doc(f) | ||
973 | f.seek(0) | ||
974 | |||
975 | if doc.documentNamespace in visited_docs: | ||
976 | return | ||
977 | |||
978 | bb.note("Adding SPDX document %s" % path) | ||
979 | visited_docs.add(doc.documentNamespace) | ||
980 | info = tar.gettarinfo(fileobj=f) | ||
981 | |||
982 | info.name = doc.name + ".spdx.json" | ||
983 | info.uid = 0 | ||
984 | info.gid = 0 | ||
985 | info.uname = "root" | ||
986 | info.gname = "root" | ||
987 | |||
988 | if source_date_epoch is not None and info.mtime > int(source_date_epoch): | ||
989 | info.mtime = int(source_date_epoch) | ||
990 | |||
991 | tar.addfile(info, f) | ||
992 | |||
993 | index["documents"].append({ | ||
994 | "filename": info.name, | ||
995 | "documentNamespace": doc.documentNamespace, | ||
996 | "sha1": sha1, | ||
997 | }) | ||
998 | |||
999 | for ref in doc.externalDocumentRefs: | ||
1000 | ref_path = deploy_dir_spdx / "by-namespace" / ref.spdxDocument.replace("/", "_") | ||
1001 | collect_spdx_document(ref_path) | ||
1002 | |||
1003 | collect_spdx_document(image_spdx_path) | ||
1004 | |||
1005 | index["documents"].sort(key=lambda x: x["filename"]) | ||
1006 | |||
1007 | index_str = io.BytesIO(json.dumps( | ||
1008 | index, | ||
1009 | sort_keys=True, | ||
1010 | indent=get_json_indent(d), | ||
1011 | ).encode("utf-8")) | ||
1012 | |||
1013 | info = tarfile.TarInfo() | ||
1014 | info.name = "index.json" | ||
1015 | info.size = len(index_str.getvalue()) | ||
1016 | info.uid = 0 | ||
1017 | info.gid = 0 | ||
1018 | info.uname = "root" | ||
1019 | info.gname = "root" | ||
1020 | |||
1021 | tar.addfile(info, fileobj=index_str) | ||
1022 | |||
1023 | spdx_index_path = rootfs_deploydir / (rootfs_name + ".spdx.index.json") | ||
1024 | with spdx_index_path.open("w") as f: | ||
1025 | json.dump(index, f, sort_keys=True, indent=get_json_indent(d)) | ||
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass index af6afcc653..19c6c0ff0b 100644 --- a/meta/classes/create-spdx.bbclass +++ b/meta/classes/create-spdx.bbclass | |||
@@ -3,1023 +3,6 @@ | |||
3 | # | 3 | # |
4 | # SPDX-License-Identifier: GPL-2.0-only | 4 | # SPDX-License-Identifier: GPL-2.0-only |
5 | # | 5 | # |
6 | 6 | # Include this class when you don't care what version of SPDX you get; it will | |
7 | DEPLOY_DIR_SPDX ??= "${DEPLOY_DIR}/spdx/${MACHINE}" | 7 | # be updated to the latest stable version that is supported |
8 | 8 | inherit create-spdx-2.2 | |
9 | # The product name that the CVE database uses. Defaults to BPN, but may need to | ||
10 | # be overriden per recipe (for example tiff.bb sets CVE_PRODUCT=libtiff). | ||
11 | CVE_PRODUCT ??= "${BPN}" | ||
12 | CVE_VERSION ??= "${PV}" | ||
13 | |||
14 | SPDXDIR ??= "${WORKDIR}/spdx" | ||
15 | SPDXDEPLOY = "${SPDXDIR}/deploy" | ||
16 | SPDXWORK = "${SPDXDIR}/work" | ||
17 | |||
18 | SPDX_TOOL_NAME ??= "oe-spdx-creator" | ||
19 | SPDX_TOOL_VERSION ??= "1.0" | ||
20 | |||
21 | SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy" | ||
22 | |||
23 | SPDX_INCLUDE_SOURCES ??= "0" | ||
24 | SPDX_ARCHIVE_SOURCES ??= "0" | ||
25 | SPDX_ARCHIVE_PACKAGED ??= "0" | ||
26 | |||
27 | SPDX_UUID_NAMESPACE ??= "sbom.openembedded.org" | ||
28 | SPDX_NAMESPACE_PREFIX ??= "http://spdx.org/spdxdoc" | ||
29 | SPDX_PRETTY ??= "0" | ||
30 | |||
31 | SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json" | ||
32 | |||
33 | SPDX_ORG ??= "OpenEmbedded ()" | ||
34 | SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}" | ||
35 | SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created from \ | ||
36 | this recipe. For SPDX documents create using this class during the build, this \ | ||
37 | is the contact information for the person or organization who is doing the \ | ||
38 | build." | ||
39 | |||
40 | def extract_licenses(filename): | ||
41 | import re | ||
42 | |||
43 | lic_regex = re.compile(rb'^\W*SPDX-License-Identifier:\s*([ \w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE) | ||
44 | |||
45 | try: | ||
46 | with open(filename, 'rb') as f: | ||
47 | size = min(15000, os.stat(filename).st_size) | ||
48 | txt = f.read(size) | ||
49 | licenses = re.findall(lic_regex, txt) | ||
50 | if licenses: | ||
51 | ascii_licenses = [lic.decode('ascii') for lic in licenses] | ||
52 | return ascii_licenses | ||
53 | except Exception as e: | ||
54 | bb.warn(f"Exception reading {filename}: {e}") | ||
55 | return None | ||
56 | |||
57 | def get_doc_namespace(d, doc): | ||
58 | import uuid | ||
59 | namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS, d.getVar("SPDX_UUID_NAMESPACE")) | ||
60 | return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name, str(uuid.uuid5(namespace_uuid, doc.name))) | ||
61 | |||
62 | def create_annotation(d, comment): | ||
63 | from datetime import datetime, timezone | ||
64 | |||
65 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | ||
66 | annotation = oe.spdx.SPDXAnnotation() | ||
67 | annotation.annotationDate = creation_time | ||
68 | annotation.annotationType = "OTHER" | ||
69 | annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) | ||
70 | annotation.comment = comment | ||
71 | return annotation | ||
72 | |||
73 | def recipe_spdx_is_native(d, recipe): | ||
74 | return any(a.annotationType == "OTHER" and | ||
75 | a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"), d.getVar("SPDX_TOOL_VERSION")) and | ||
76 | a.comment == "isNative" for a in recipe.annotations) | ||
77 | |||
78 | def is_work_shared_spdx(d): | ||
79 | return bb.data.inherits_class('kernel', d) or ('work-shared' in d.getVar('WORKDIR')) | ||
80 | |||
81 | def get_json_indent(d): | ||
82 | if d.getVar("SPDX_PRETTY") == "1": | ||
83 | return 2 | ||
84 | return None | ||
85 | |||
86 | python() { | ||
87 | import json | ||
88 | if d.getVar("SPDX_LICENSE_DATA"): | ||
89 | return | ||
90 | |||
91 | with open(d.getVar("SPDX_LICENSES"), "r") as f: | ||
92 | data = json.load(f) | ||
93 | # Transform the license array to a dictionary | ||
94 | data["licenses"] = {l["licenseId"]: l for l in data["licenses"]} | ||
95 | d.setVar("SPDX_LICENSE_DATA", data) | ||
96 | } | ||
97 | |||
98 | def convert_license_to_spdx(lic, document, d, existing={}): | ||
99 | from pathlib import Path | ||
100 | import oe.spdx | ||
101 | |||
102 | license_data = d.getVar("SPDX_LICENSE_DATA") | ||
103 | extracted = {} | ||
104 | |||
105 | def add_extracted_license(ident, name): | ||
106 | nonlocal document | ||
107 | |||
108 | if name in extracted: | ||
109 | return | ||
110 | |||
111 | extracted_info = oe.spdx.SPDXExtractedLicensingInfo() | ||
112 | extracted_info.name = name | ||
113 | extracted_info.licenseId = ident | ||
114 | extracted_info.extractedText = None | ||
115 | |||
116 | if name == "PD": | ||
117 | # Special-case this. | ||
118 | extracted_info.extractedText = "Software released to the public domain" | ||
119 | else: | ||
120 | # Seach for the license in COMMON_LICENSE_DIR and LICENSE_PATH | ||
121 | for directory in [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or '').split(): | ||
122 | try: | ||
123 | with (Path(directory) / name).open(errors="replace") as f: | ||
124 | extracted_info.extractedText = f.read() | ||
125 | break | ||
126 | except FileNotFoundError: | ||
127 | pass | ||
128 | if extracted_info.extractedText is None: | ||
129 | # If it's not SPDX or PD, then NO_GENERIC_LICENSE must be set | ||
130 | filename = d.getVarFlag('NO_GENERIC_LICENSE', name) | ||
131 | if filename: | ||
132 | filename = d.expand("${S}/" + filename) | ||
133 | with open(filename, errors="replace") as f: | ||
134 | extracted_info.extractedText = f.read() | ||
135 | else: | ||
136 | bb.error("Cannot find any text for license %s" % name) | ||
137 | |||
138 | extracted[name] = extracted_info | ||
139 | document.hasExtractedLicensingInfos.append(extracted_info) | ||
140 | |||
141 | def convert(l): | ||
142 | if l == "(" or l == ")": | ||
143 | return l | ||
144 | |||
145 | if l == "&": | ||
146 | return "AND" | ||
147 | |||
148 | if l == "|": | ||
149 | return "OR" | ||
150 | |||
151 | if l == "CLOSED": | ||
152 | return "NONE" | ||
153 | |||
154 | spdx_license = d.getVarFlag("SPDXLICENSEMAP", l) or l | ||
155 | if spdx_license in license_data["licenses"]: | ||
156 | return spdx_license | ||
157 | |||
158 | try: | ||
159 | spdx_license = existing[l] | ||
160 | except KeyError: | ||
161 | spdx_license = "LicenseRef-" + l | ||
162 | add_extracted_license(spdx_license, l) | ||
163 | |||
164 | return spdx_license | ||
165 | |||
166 | lic_split = lic.replace("(", " ( ").replace(")", " ) ").split() | ||
167 | |||
168 | return ' '.join(convert(l) for l in lic_split) | ||
169 | |||
170 | def process_sources(d): | ||
171 | pn = d.getVar('PN') | ||
172 | assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split() | ||
173 | if pn in assume_provided: | ||
174 | for p in d.getVar("PROVIDES").split(): | ||
175 | if p != pn: | ||
176 | pn = p | ||
177 | break | ||
178 | |||
179 | # glibc-locale: do_fetch, do_unpack and do_patch tasks have been deleted, | ||
180 | # so avoid archiving source here. | ||
181 | if pn.startswith('glibc-locale'): | ||
182 | return False | ||
183 | if d.getVar('PN') == "libtool-cross": | ||
184 | return False | ||
185 | if d.getVar('PN') == "libgcc-initial": | ||
186 | return False | ||
187 | if d.getVar('PN') == "shadow-sysroot": | ||
188 | return False | ||
189 | |||
190 | # We just archive gcc-source for all the gcc related recipes | ||
191 | if d.getVar('BPN') in ['gcc', 'libgcc']: | ||
192 | bb.debug(1, 'spdx: There is bug in scan of %s is, do nothing' % pn) | ||
193 | return False | ||
194 | |||
195 | return True | ||
196 | |||
197 | |||
198 | def add_package_files(d, doc, spdx_pkg, topdir, get_spdxid, get_types, *, archive=None, ignore_dirs=[], ignore_top_level_dirs=[]): | ||
199 | from pathlib import Path | ||
200 | import oe.spdx | ||
201 | import hashlib | ||
202 | |||
203 | source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") | ||
204 | if source_date_epoch: | ||
205 | source_date_epoch = int(source_date_epoch) | ||
206 | |||
207 | sha1s = [] | ||
208 | spdx_files = [] | ||
209 | |||
210 | file_counter = 1 | ||
211 | for subdir, dirs, files in os.walk(topdir): | ||
212 | dirs[:] = [d for d in dirs if d not in ignore_dirs] | ||
213 | if subdir == str(topdir): | ||
214 | dirs[:] = [d for d in dirs if d not in ignore_top_level_dirs] | ||
215 | |||
216 | for file in files: | ||
217 | filepath = Path(subdir) / file | ||
218 | filename = str(filepath.relative_to(topdir)) | ||
219 | |||
220 | if not filepath.is_symlink() and filepath.is_file(): | ||
221 | spdx_file = oe.spdx.SPDXFile() | ||
222 | spdx_file.SPDXID = get_spdxid(file_counter) | ||
223 | for t in get_types(filepath): | ||
224 | spdx_file.fileTypes.append(t) | ||
225 | spdx_file.fileName = filename | ||
226 | |||
227 | if archive is not None: | ||
228 | with filepath.open("rb") as f: | ||
229 | info = archive.gettarinfo(fileobj=f) | ||
230 | info.name = filename | ||
231 | info.uid = 0 | ||
232 | info.gid = 0 | ||
233 | info.uname = "root" | ||
234 | info.gname = "root" | ||
235 | |||
236 | if source_date_epoch is not None and info.mtime > source_date_epoch: | ||
237 | info.mtime = source_date_epoch | ||
238 | |||
239 | archive.addfile(info, f) | ||
240 | |||
241 | sha1 = bb.utils.sha1_file(filepath) | ||
242 | sha1s.append(sha1) | ||
243 | spdx_file.checksums.append(oe.spdx.SPDXChecksum( | ||
244 | algorithm="SHA1", | ||
245 | checksumValue=sha1, | ||
246 | )) | ||
247 | spdx_file.checksums.append(oe.spdx.SPDXChecksum( | ||
248 | algorithm="SHA256", | ||
249 | checksumValue=bb.utils.sha256_file(filepath), | ||
250 | )) | ||
251 | |||
252 | if "SOURCE" in spdx_file.fileTypes: | ||
253 | extracted_lics = extract_licenses(filepath) | ||
254 | if extracted_lics: | ||
255 | spdx_file.licenseInfoInFiles = extracted_lics | ||
256 | |||
257 | doc.files.append(spdx_file) | ||
258 | doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file) | ||
259 | spdx_pkg.hasFiles.append(spdx_file.SPDXID) | ||
260 | |||
261 | spdx_files.append(spdx_file) | ||
262 | |||
263 | file_counter += 1 | ||
264 | |||
265 | sha1s.sort() | ||
266 | verifier = hashlib.sha1() | ||
267 | for v in sha1s: | ||
268 | verifier.update(v.encode("utf-8")) | ||
269 | spdx_pkg.packageVerificationCode.packageVerificationCodeValue = verifier.hexdigest() | ||
270 | |||
271 | return spdx_files | ||
272 | |||
273 | |||
274 | def add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources): | ||
275 | from pathlib import Path | ||
276 | import hashlib | ||
277 | import oe.packagedata | ||
278 | import oe.spdx | ||
279 | |||
280 | debug_search_paths = [ | ||
281 | Path(d.getVar('PKGD')), | ||
282 | Path(d.getVar('STAGING_DIR_TARGET')), | ||
283 | Path(d.getVar('STAGING_DIR_NATIVE')), | ||
284 | Path(d.getVar('STAGING_KERNEL_DIR')), | ||
285 | ] | ||
286 | |||
287 | pkg_data = oe.packagedata.read_subpkgdata_extended(package, d) | ||
288 | |||
289 | if pkg_data is None: | ||
290 | return | ||
291 | |||
292 | for file_path, file_data in pkg_data["files_info"].items(): | ||
293 | if not "debugsrc" in file_data: | ||
294 | continue | ||
295 | |||
296 | for pkg_file in package_files: | ||
297 | if file_path.lstrip("/") == pkg_file.fileName.lstrip("/"): | ||
298 | break | ||
299 | else: | ||
300 | bb.fatal("No package file found for %s" % str(file_path)) | ||
301 | continue | ||
302 | |||
303 | for debugsrc in file_data["debugsrc"]: | ||
304 | ref_id = "NOASSERTION" | ||
305 | for search in debug_search_paths: | ||
306 | if debugsrc.startswith("/usr/src/kernel"): | ||
307 | debugsrc_path = search / debugsrc.replace('/usr/src/kernel/', '') | ||
308 | else: | ||
309 | debugsrc_path = search / debugsrc.lstrip("/") | ||
310 | if not debugsrc_path.exists(): | ||
311 | continue | ||
312 | |||
313 | file_sha256 = bb.utils.sha256_file(debugsrc_path) | ||
314 | |||
315 | if file_sha256 in sources: | ||
316 | source_file = sources[file_sha256] | ||
317 | |||
318 | doc_ref = package_doc.find_external_document_ref(source_file.doc.documentNamespace) | ||
319 | if doc_ref is None: | ||
320 | doc_ref = oe.spdx.SPDXExternalDocumentRef() | ||
321 | doc_ref.externalDocumentId = "DocumentRef-dependency-" + source_file.doc.name | ||
322 | doc_ref.spdxDocument = source_file.doc.documentNamespace | ||
323 | doc_ref.checksum.algorithm = "SHA1" | ||
324 | doc_ref.checksum.checksumValue = source_file.doc_sha1 | ||
325 | package_doc.externalDocumentRefs.append(doc_ref) | ||
326 | |||
327 | ref_id = "%s:%s" % (doc_ref.externalDocumentId, source_file.file.SPDXID) | ||
328 | else: | ||
329 | bb.debug(1, "Debug source %s with SHA256 %s not found in any dependency" % (str(debugsrc_path), file_sha256)) | ||
330 | break | ||
331 | else: | ||
332 | bb.debug(1, "Debug source %s not found" % debugsrc) | ||
333 | |||
334 | package_doc.add_relationship(pkg_file, "GENERATED_FROM", ref_id, comment=debugsrc) | ||
335 | |||
336 | def collect_dep_recipes(d, doc, spdx_recipe): | ||
337 | from pathlib import Path | ||
338 | import oe.sbom | ||
339 | import oe.spdx | ||
340 | |||
341 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) | ||
342 | |||
343 | dep_recipes = [] | ||
344 | taskdepdata = d.getVar("BB_TASKDEPDATA", False) | ||
345 | deps = sorted(set( | ||
346 | dep[0] for dep in taskdepdata.values() if | ||
347 | dep[1] == "do_create_spdx" and dep[0] != d.getVar("PN") | ||
348 | )) | ||
349 | for dep_pn in deps: | ||
350 | dep_recipe_path = deploy_dir_spdx / "recipes" / ("recipe-%s.spdx.json" % dep_pn) | ||
351 | |||
352 | spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_recipe_path) | ||
353 | |||
354 | for pkg in spdx_dep_doc.packages: | ||
355 | if pkg.name == dep_pn: | ||
356 | spdx_dep_recipe = pkg | ||
357 | break | ||
358 | else: | ||
359 | continue | ||
360 | |||
361 | dep_recipes.append(oe.sbom.DepRecipe(spdx_dep_doc, spdx_dep_sha1, spdx_dep_recipe)) | ||
362 | |||
363 | dep_recipe_ref = oe.spdx.SPDXExternalDocumentRef() | ||
364 | dep_recipe_ref.externalDocumentId = "DocumentRef-dependency-" + spdx_dep_doc.name | ||
365 | dep_recipe_ref.spdxDocument = spdx_dep_doc.documentNamespace | ||
366 | dep_recipe_ref.checksum.algorithm = "SHA1" | ||
367 | dep_recipe_ref.checksum.checksumValue = spdx_dep_sha1 | ||
368 | |||
369 | doc.externalDocumentRefs.append(dep_recipe_ref) | ||
370 | |||
371 | doc.add_relationship( | ||
372 | "%s:%s" % (dep_recipe_ref.externalDocumentId, spdx_dep_recipe.SPDXID), | ||
373 | "BUILD_DEPENDENCY_OF", | ||
374 | spdx_recipe | ||
375 | ) | ||
376 | |||
377 | return dep_recipes | ||
378 | |||
379 | collect_dep_recipes[vardepsexclude] += "BB_TASKDEPDATA" | ||
380 | |||
381 | |||
382 | def collect_dep_sources(d, dep_recipes): | ||
383 | import oe.sbom | ||
384 | |||
385 | sources = {} | ||
386 | for dep in dep_recipes: | ||
387 | # Don't collect sources from native recipes as they | ||
388 | # match non-native sources also. | ||
389 | if recipe_spdx_is_native(d, dep.recipe): | ||
390 | continue | ||
391 | recipe_files = set(dep.recipe.hasFiles) | ||
392 | |||
393 | for spdx_file in dep.doc.files: | ||
394 | if spdx_file.SPDXID not in recipe_files: | ||
395 | continue | ||
396 | |||
397 | if "SOURCE" in spdx_file.fileTypes: | ||
398 | for checksum in spdx_file.checksums: | ||
399 | if checksum.algorithm == "SHA256": | ||
400 | sources[checksum.checksumValue] = oe.sbom.DepSource(dep.doc, dep.doc_sha1, dep.recipe, spdx_file) | ||
401 | break | ||
402 | |||
403 | return sources | ||
404 | |||
405 | |||
406 | python do_create_spdx() { | ||
407 | from datetime import datetime, timezone | ||
408 | import oe.sbom | ||
409 | import oe.spdx | ||
410 | import uuid | ||
411 | from pathlib import Path | ||
412 | from contextlib import contextmanager | ||
413 | import oe.cve_check | ||
414 | |||
415 | @contextmanager | ||
416 | def optional_tarfile(name, guard, mode="w"): | ||
417 | import tarfile | ||
418 | import bb.compress.zstd | ||
419 | |||
420 | num_threads = int(d.getVar("BB_NUMBER_THREADS")) | ||
421 | |||
422 | if guard: | ||
423 | name.parent.mkdir(parents=True, exist_ok=True) | ||
424 | with bb.compress.zstd.open(name, mode=mode + "b", num_threads=num_threads) as f: | ||
425 | with tarfile.open(fileobj=f, mode=mode + "|") as tf: | ||
426 | yield tf | ||
427 | else: | ||
428 | yield None | ||
429 | |||
430 | |||
431 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) | ||
432 | spdx_workdir = Path(d.getVar("SPDXWORK")) | ||
433 | include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1" | ||
434 | archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1" | ||
435 | archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1" | ||
436 | |||
437 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | ||
438 | |||
439 | doc = oe.spdx.SPDXDocument() | ||
440 | |||
441 | doc.name = "recipe-" + d.getVar("PN") | ||
442 | doc.documentNamespace = get_doc_namespace(d, doc) | ||
443 | doc.creationInfo.created = creation_time | ||
444 | doc.creationInfo.comment = "This document was created by analyzing recipe files during the build." | ||
445 | doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] | ||
446 | doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") | ||
447 | doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) | ||
448 | doc.creationInfo.creators.append("Person: N/A ()") | ||
449 | |||
450 | recipe = oe.spdx.SPDXPackage() | ||
451 | recipe.name = d.getVar("PN") | ||
452 | recipe.versionInfo = d.getVar("PV") | ||
453 | recipe.SPDXID = oe.sbom.get_recipe_spdxid(d) | ||
454 | recipe.supplier = d.getVar("SPDX_SUPPLIER") | ||
455 | if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d): | ||
456 | recipe.annotations.append(create_annotation(d, "isNative")) | ||
457 | |||
458 | for s in d.getVar('SRC_URI').split(): | ||
459 | if not s.startswith("file://"): | ||
460 | s = s.split(';')[0] | ||
461 | recipe.downloadLocation = s | ||
462 | break | ||
463 | else: | ||
464 | recipe.downloadLocation = "NOASSERTION" | ||
465 | |||
466 | homepage = d.getVar("HOMEPAGE") | ||
467 | if homepage: | ||
468 | recipe.homepage = homepage | ||
469 | |||
470 | license = d.getVar("LICENSE") | ||
471 | if license: | ||
472 | recipe.licenseDeclared = convert_license_to_spdx(license, doc, d) | ||
473 | |||
474 | summary = d.getVar("SUMMARY") | ||
475 | if summary: | ||
476 | recipe.summary = summary | ||
477 | |||
478 | description = d.getVar("DESCRIPTION") | ||
479 | if description: | ||
480 | recipe.description = description | ||
481 | |||
482 | # Some CVEs may be patched during the build process without incrementing the version number, | ||
483 | # so querying for CVEs based on the CPE id can lead to false positives. To account for this, | ||
484 | # save the CVEs fixed by patches to source information field in the SPDX. | ||
485 | patched_cves = oe.cve_check.get_patched_cves(d) | ||
486 | patched_cves = list(patched_cves) | ||
487 | patched_cves = ' '.join(patched_cves) | ||
488 | if patched_cves: | ||
489 | recipe.sourceInfo = "CVEs fixed: " + patched_cves | ||
490 | |||
491 | cpe_ids = oe.cve_check.get_cpe_ids(d.getVar("CVE_PRODUCT"), d.getVar("CVE_VERSION")) | ||
492 | if cpe_ids: | ||
493 | for cpe_id in cpe_ids: | ||
494 | cpe = oe.spdx.SPDXExternalReference() | ||
495 | cpe.referenceCategory = "SECURITY" | ||
496 | cpe.referenceType = "http://spdx.org/rdf/references/cpe23Type" | ||
497 | cpe.referenceLocator = cpe_id | ||
498 | recipe.externalRefs.append(cpe) | ||
499 | |||
500 | doc.packages.append(recipe) | ||
501 | doc.add_relationship(doc, "DESCRIBES", recipe) | ||
502 | |||
503 | if process_sources(d) and include_sources: | ||
504 | recipe_archive = deploy_dir_spdx / "recipes" / (doc.name + ".tar.zst") | ||
505 | with optional_tarfile(recipe_archive, archive_sources) as archive: | ||
506 | spdx_get_src(d) | ||
507 | |||
508 | add_package_files( | ||
509 | d, | ||
510 | doc, | ||
511 | recipe, | ||
512 | spdx_workdir, | ||
513 | lambda file_counter: "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), file_counter), | ||
514 | lambda filepath: ["SOURCE"], | ||
515 | ignore_dirs=[".git"], | ||
516 | ignore_top_level_dirs=["temp"], | ||
517 | archive=archive, | ||
518 | ) | ||
519 | |||
520 | if archive is not None: | ||
521 | recipe.packageFileName = str(recipe_archive.name) | ||
522 | |||
523 | dep_recipes = collect_dep_recipes(d, doc, recipe) | ||
524 | |||
525 | doc_sha1 = oe.sbom.write_doc(d, doc, "recipes", indent=get_json_indent(d)) | ||
526 | dep_recipes.append(oe.sbom.DepRecipe(doc, doc_sha1, recipe)) | ||
527 | |||
528 | recipe_ref = oe.spdx.SPDXExternalDocumentRef() | ||
529 | recipe_ref.externalDocumentId = "DocumentRef-recipe-" + recipe.name | ||
530 | recipe_ref.spdxDocument = doc.documentNamespace | ||
531 | recipe_ref.checksum.algorithm = "SHA1" | ||
532 | recipe_ref.checksum.checksumValue = doc_sha1 | ||
533 | |||
534 | sources = collect_dep_sources(d, dep_recipes) | ||
535 | found_licenses = {license.name:recipe_ref.externalDocumentId + ":" + license.licenseId for license in doc.hasExtractedLicensingInfos} | ||
536 | |||
537 | if not recipe_spdx_is_native(d, recipe): | ||
538 | bb.build.exec_func("read_subpackage_metadata", d) | ||
539 | |||
540 | pkgdest = Path(d.getVar("PKGDEST")) | ||
541 | for package in d.getVar("PACKAGES").split(): | ||
542 | if not oe.packagedata.packaged(package, d): | ||
543 | continue | ||
544 | |||
545 | package_doc = oe.spdx.SPDXDocument() | ||
546 | pkg_name = d.getVar("PKG:%s" % package) or package | ||
547 | package_doc.name = pkg_name | ||
548 | package_doc.documentNamespace = get_doc_namespace(d, package_doc) | ||
549 | package_doc.creationInfo.created = creation_time | ||
550 | package_doc.creationInfo.comment = "This document was created by analyzing packages created during the build." | ||
551 | package_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] | ||
552 | package_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") | ||
553 | package_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) | ||
554 | package_doc.creationInfo.creators.append("Person: N/A ()") | ||
555 | package_doc.externalDocumentRefs.append(recipe_ref) | ||
556 | |||
557 | package_license = d.getVar("LICENSE:%s" % package) or d.getVar("LICENSE") | ||
558 | |||
559 | spdx_package = oe.spdx.SPDXPackage() | ||
560 | |||
561 | spdx_package.SPDXID = oe.sbom.get_package_spdxid(pkg_name) | ||
562 | spdx_package.name = pkg_name | ||
563 | spdx_package.versionInfo = d.getVar("PV") | ||
564 | spdx_package.licenseDeclared = convert_license_to_spdx(package_license, package_doc, d, found_licenses) | ||
565 | spdx_package.supplier = d.getVar("SPDX_SUPPLIER") | ||
566 | |||
567 | package_doc.packages.append(spdx_package) | ||
568 | |||
569 | package_doc.add_relationship(spdx_package, "GENERATED_FROM", "%s:%s" % (recipe_ref.externalDocumentId, recipe.SPDXID)) | ||
570 | package_doc.add_relationship(package_doc, "DESCRIBES", spdx_package) | ||
571 | |||
572 | package_archive = deploy_dir_spdx / "packages" / (package_doc.name + ".tar.zst") | ||
573 | with optional_tarfile(package_archive, archive_packaged) as archive: | ||
574 | package_files = add_package_files( | ||
575 | d, | ||
576 | package_doc, | ||
577 | spdx_package, | ||
578 | pkgdest / package, | ||
579 | lambda file_counter: oe.sbom.get_packaged_file_spdxid(pkg_name, file_counter), | ||
580 | lambda filepath: ["BINARY"], | ||
581 | ignore_top_level_dirs=['CONTROL', 'DEBIAN'], | ||
582 | archive=archive, | ||
583 | ) | ||
584 | |||
585 | if archive is not None: | ||
586 | spdx_package.packageFileName = str(package_archive.name) | ||
587 | |||
588 | add_package_sources_from_debug(d, package_doc, spdx_package, package, package_files, sources) | ||
589 | |||
590 | oe.sbom.write_doc(d, package_doc, "packages", indent=get_json_indent(d)) | ||
591 | } | ||
592 | # NOTE: depending on do_unpack is a hack that is necessary to get it's dependencies for archive the source | ||
593 | addtask do_create_spdx after do_package do_packagedata do_unpack before do_populate_sdk do_build do_rm_work | ||
594 | |||
595 | SSTATETASKS += "do_create_spdx" | ||
596 | do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}" | ||
597 | do_create_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" | ||
598 | |||
599 | python do_create_spdx_setscene () { | ||
600 | sstate_setscene(d) | ||
601 | } | ||
602 | addtask do_create_spdx_setscene | ||
603 | |||
604 | do_create_spdx[dirs] = "${SPDXWORK}" | ||
605 | do_create_spdx[cleandirs] = "${SPDXDEPLOY} ${SPDXWORK}" | ||
606 | do_create_spdx[depends] += "${PATCHDEPENDENCY}" | ||
607 | do_create_spdx[deptask] = "do_create_spdx" | ||
608 | |||
609 | def collect_package_providers(d): | ||
610 | from pathlib import Path | ||
611 | import oe.sbom | ||
612 | import oe.spdx | ||
613 | import json | ||
614 | |||
615 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) | ||
616 | |||
617 | providers = {} | ||
618 | |||
619 | taskdepdata = d.getVar("BB_TASKDEPDATA", False) | ||
620 | deps = sorted(set( | ||
621 | dep[0] for dep in taskdepdata.values() if dep[0] != d.getVar("PN") | ||
622 | )) | ||
623 | deps.append(d.getVar("PN")) | ||
624 | |||
625 | for dep_pn in deps: | ||
626 | recipe_data = oe.packagedata.read_pkgdata(dep_pn, d) | ||
627 | |||
628 | for pkg in recipe_data.get("PACKAGES", "").split(): | ||
629 | |||
630 | pkg_data = oe.packagedata.read_subpkgdata_dict(pkg, d) | ||
631 | rprovides = set(n for n, _ in bb.utils.explode_dep_versions2(pkg_data.get("RPROVIDES", "")).items()) | ||
632 | rprovides.add(pkg) | ||
633 | |||
634 | for r in rprovides: | ||
635 | providers[r] = pkg | ||
636 | |||
637 | return providers | ||
638 | |||
639 | collect_package_providers[vardepsexclude] += "BB_TASKDEPDATA" | ||
640 | |||
641 | python do_create_runtime_spdx() { | ||
642 | from datetime import datetime, timezone | ||
643 | import oe.sbom | ||
644 | import oe.spdx | ||
645 | import oe.packagedata | ||
646 | from pathlib import Path | ||
647 | |||
648 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) | ||
649 | spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY")) | ||
650 | is_native = bb.data.inherits_class("native", d) or bb.data.inherits_class("cross", d) | ||
651 | |||
652 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | ||
653 | |||
654 | providers = collect_package_providers(d) | ||
655 | |||
656 | if not is_native: | ||
657 | bb.build.exec_func("read_subpackage_metadata", d) | ||
658 | |||
659 | dep_package_cache = {} | ||
660 | |||
661 | pkgdest = Path(d.getVar("PKGDEST")) | ||
662 | for package in d.getVar("PACKAGES").split(): | ||
663 | localdata = bb.data.createCopy(d) | ||
664 | pkg_name = d.getVar("PKG:%s" % package) or package | ||
665 | localdata.setVar("PKG", pkg_name) | ||
666 | localdata.setVar('OVERRIDES', d.getVar("OVERRIDES", False) + ":" + package) | ||
667 | |||
668 | if not oe.packagedata.packaged(package, localdata): | ||
669 | continue | ||
670 | |||
671 | pkg_spdx_path = deploy_dir_spdx / "packages" / (pkg_name + ".spdx.json") | ||
672 | |||
673 | package_doc, package_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) | ||
674 | |||
675 | for p in package_doc.packages: | ||
676 | if p.name == pkg_name: | ||
677 | spdx_package = p | ||
678 | break | ||
679 | else: | ||
680 | bb.fatal("Package '%s' not found in %s" % (pkg_name, pkg_spdx_path)) | ||
681 | |||
682 | runtime_doc = oe.spdx.SPDXDocument() | ||
683 | runtime_doc.name = "runtime-" + pkg_name | ||
684 | runtime_doc.documentNamespace = get_doc_namespace(localdata, runtime_doc) | ||
685 | runtime_doc.creationInfo.created = creation_time | ||
686 | runtime_doc.creationInfo.comment = "This document was created by analyzing package runtime dependencies." | ||
687 | runtime_doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] | ||
688 | runtime_doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") | ||
689 | runtime_doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) | ||
690 | runtime_doc.creationInfo.creators.append("Person: N/A ()") | ||
691 | |||
692 | package_ref = oe.spdx.SPDXExternalDocumentRef() | ||
693 | package_ref.externalDocumentId = "DocumentRef-package-" + package | ||
694 | package_ref.spdxDocument = package_doc.documentNamespace | ||
695 | package_ref.checksum.algorithm = "SHA1" | ||
696 | package_ref.checksum.checksumValue = package_doc_sha1 | ||
697 | |||
698 | runtime_doc.externalDocumentRefs.append(package_ref) | ||
699 | |||
700 | runtime_doc.add_relationship( | ||
701 | runtime_doc.SPDXID, | ||
702 | "AMENDS", | ||
703 | "%s:%s" % (package_ref.externalDocumentId, package_doc.SPDXID) | ||
704 | ) | ||
705 | |||
706 | deps = bb.utils.explode_dep_versions2(localdata.getVar("RDEPENDS") or "") | ||
707 | seen_deps = set() | ||
708 | for dep, _ in deps.items(): | ||
709 | if dep in seen_deps: | ||
710 | continue | ||
711 | |||
712 | if dep not in providers: | ||
713 | continue | ||
714 | |||
715 | dep = providers[dep] | ||
716 | |||
717 | if not oe.packagedata.packaged(dep, localdata): | ||
718 | continue | ||
719 | |||
720 | dep_pkg_data = oe.packagedata.read_subpkgdata_dict(dep, d) | ||
721 | dep_pkg = dep_pkg_data["PKG"] | ||
722 | |||
723 | if dep in dep_package_cache: | ||
724 | (dep_spdx_package, dep_package_ref) = dep_package_cache[dep] | ||
725 | else: | ||
726 | dep_path = deploy_dir_spdx / "packages" / ("%s.spdx.json" % dep_pkg) | ||
727 | |||
728 | spdx_dep_doc, spdx_dep_sha1 = oe.sbom.read_doc(dep_path) | ||
729 | |||
730 | for pkg in spdx_dep_doc.packages: | ||
731 | if pkg.name == dep_pkg: | ||
732 | dep_spdx_package = pkg | ||
733 | break | ||
734 | else: | ||
735 | bb.fatal("Package '%s' not found in %s" % (dep_pkg, dep_path)) | ||
736 | |||
737 | dep_package_ref = oe.spdx.SPDXExternalDocumentRef() | ||
738 | dep_package_ref.externalDocumentId = "DocumentRef-runtime-dependency-" + spdx_dep_doc.name | ||
739 | dep_package_ref.spdxDocument = spdx_dep_doc.documentNamespace | ||
740 | dep_package_ref.checksum.algorithm = "SHA1" | ||
741 | dep_package_ref.checksum.checksumValue = spdx_dep_sha1 | ||
742 | |||
743 | dep_package_cache[dep] = (dep_spdx_package, dep_package_ref) | ||
744 | |||
745 | runtime_doc.externalDocumentRefs.append(dep_package_ref) | ||
746 | |||
747 | runtime_doc.add_relationship( | ||
748 | "%s:%s" % (dep_package_ref.externalDocumentId, dep_spdx_package.SPDXID), | ||
749 | "RUNTIME_DEPENDENCY_OF", | ||
750 | "%s:%s" % (package_ref.externalDocumentId, spdx_package.SPDXID) | ||
751 | ) | ||
752 | seen_deps.add(dep) | ||
753 | |||
754 | oe.sbom.write_doc(d, runtime_doc, "runtime", spdx_deploy, indent=get_json_indent(d)) | ||
755 | } | ||
756 | |||
757 | addtask do_create_runtime_spdx after do_create_spdx before do_build do_rm_work | ||
758 | SSTATETASKS += "do_create_runtime_spdx" | ||
759 | do_create_runtime_spdx[sstate-inputdirs] = "${SPDXRUNTIMEDEPLOY}" | ||
760 | do_create_runtime_spdx[sstate-outputdirs] = "${DEPLOY_DIR_SPDX}" | ||
761 | |||
762 | python do_create_runtime_spdx_setscene () { | ||
763 | sstate_setscene(d) | ||
764 | } | ||
765 | addtask do_create_runtime_spdx_setscene | ||
766 | |||
767 | do_create_runtime_spdx[dirs] = "${SPDXRUNTIMEDEPLOY}" | ||
768 | do_create_runtime_spdx[cleandirs] = "${SPDXRUNTIMEDEPLOY}" | ||
769 | do_create_runtime_spdx[rdeptask] = "do_create_spdx" | ||
770 | |||
771 | def spdx_get_src(d): | ||
772 | """ | ||
773 | save patched source of the recipe in SPDX_WORKDIR. | ||
774 | """ | ||
775 | import shutil | ||
776 | spdx_workdir = d.getVar('SPDXWORK') | ||
777 | spdx_sysroot_native = d.getVar('STAGING_DIR_NATIVE') | ||
778 | pn = d.getVar('PN') | ||
779 | |||
780 | workdir = d.getVar("WORKDIR") | ||
781 | |||
782 | try: | ||
783 | # The kernel class functions require it to be on work-shared, so we dont change WORKDIR | ||
784 | if not is_work_shared_spdx(d): | ||
785 | # Change the WORKDIR to make do_unpack do_patch run in another dir. | ||
786 | d.setVar('WORKDIR', spdx_workdir) | ||
787 | # Restore the original path to recipe's native sysroot (it's relative to WORKDIR). | ||
788 | d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) | ||
789 | |||
790 | # The changed 'WORKDIR' also caused 'B' changed, create dir 'B' for the | ||
791 | # possibly requiring of the following tasks (such as some recipes's | ||
792 | # do_patch required 'B' existed). | ||
793 | bb.utils.mkdirhier(d.getVar('B')) | ||
794 | |||
795 | bb.build.exec_func('do_unpack', d) | ||
796 | # Copy source of kernel to spdx_workdir | ||
797 | if is_work_shared_spdx(d): | ||
798 | d.setVar('WORKDIR', spdx_workdir) | ||
799 | d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native) | ||
800 | src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" + d.getVar('PV') + "-" + d.getVar('PR') | ||
801 | bb.utils.mkdirhier(src_dir) | ||
802 | if bb.data.inherits_class('kernel',d): | ||
803 | share_src = d.getVar('STAGING_KERNEL_DIR') | ||
804 | cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/" | ||
805 | cmd_copy_kernel_result = os.popen(cmd_copy_share).read() | ||
806 | bb.note("cmd_copy_kernel_result = " + cmd_copy_kernel_result) | ||
807 | |||
808 | git_path = src_dir + "/.git" | ||
809 | if os.path.exists(git_path): | ||
810 | shutils.rmtree(git_path) | ||
811 | |||
812 | # Make sure gcc and kernel sources are patched only once | ||
813 | if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)): | ||
814 | bb.build.exec_func('do_patch', d) | ||
815 | |||
816 | # Some userland has no source. | ||
817 | if not os.path.exists( spdx_workdir ): | ||
818 | bb.utils.mkdirhier(spdx_workdir) | ||
819 | finally: | ||
820 | d.setVar("WORKDIR", workdir) | ||
821 | |||
822 | do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx" | ||
823 | |||
824 | ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; " | ||
825 | |||
826 | do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx" | ||
827 | POPULATE_SDK_POST_HOST_COMMAND:append:task-populate-sdk = " sdk_host_combine_spdx; " | ||
828 | POPULATE_SDK_POST_TARGET_COMMAND:append:task-populate-sdk = " sdk_target_combine_spdx; " | ||
829 | |||
830 | python image_combine_spdx() { | ||
831 | import os | ||
832 | import oe.sbom | ||
833 | from pathlib import Path | ||
834 | from oe.rootfs import image_list_installed_packages | ||
835 | |||
836 | image_name = d.getVar("IMAGE_NAME") | ||
837 | image_link_name = d.getVar("IMAGE_LINK_NAME") | ||
838 | imgdeploydir = Path(d.getVar("IMGDEPLOYDIR")) | ||
839 | img_spdxid = oe.sbom.get_image_spdxid(image_name) | ||
840 | packages = image_list_installed_packages(d) | ||
841 | |||
842 | combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages) | ||
843 | |||
844 | def make_image_link(target_path, suffix): | ||
845 | if image_link_name: | ||
846 | link = imgdeploydir / (image_link_name + suffix) | ||
847 | if link != target_path: | ||
848 | link.symlink_to(os.path.relpath(target_path, link.parent)) | ||
849 | |||
850 | image_spdx_path = imgdeploydir / (image_name + ".spdx.json") | ||
851 | make_image_link(image_spdx_path, ".spdx.json") | ||
852 | spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst") | ||
853 | make_image_link(spdx_tar_path, ".spdx.tar.zst") | ||
854 | spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json") | ||
855 | make_image_link(spdx_index_path, ".spdx.index.json") | ||
856 | } | ||
857 | |||
858 | python sdk_host_combine_spdx() { | ||
859 | sdk_combine_spdx(d, "host") | ||
860 | } | ||
861 | |||
862 | python sdk_target_combine_spdx() { | ||
863 | sdk_combine_spdx(d, "target") | ||
864 | } | ||
865 | |||
866 | def sdk_combine_spdx(d, sdk_type): | ||
867 | import oe.sbom | ||
868 | from pathlib import Path | ||
869 | from oe.sdk import sdk_list_installed_packages | ||
870 | |||
871 | sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type | ||
872 | sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR")) | ||
873 | sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name) | ||
874 | sdk_packages = sdk_list_installed_packages(d, sdk_type == "target") | ||
875 | combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages) | ||
876 | |||
877 | def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages): | ||
878 | import os | ||
879 | import oe.spdx | ||
880 | import oe.sbom | ||
881 | import io | ||
882 | import json | ||
883 | from datetime import timezone, datetime | ||
884 | from pathlib import Path | ||
885 | import tarfile | ||
886 | import bb.compress.zstd | ||
887 | |||
888 | creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") | ||
889 | deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX")) | ||
890 | source_date_epoch = d.getVar("SOURCE_DATE_EPOCH") | ||
891 | |||
892 | doc = oe.spdx.SPDXDocument() | ||
893 | doc.name = rootfs_name | ||
894 | doc.documentNamespace = get_doc_namespace(d, doc) | ||
895 | doc.creationInfo.created = creation_time | ||
896 | doc.creationInfo.comment = "This document was created by analyzing the source of the Yocto recipe during the build." | ||
897 | doc.creationInfo.licenseListVersion = d.getVar("SPDX_LICENSE_DATA")["licenseListVersion"] | ||
898 | doc.creationInfo.creators.append("Tool: OpenEmbedded Core create-spdx.bbclass") | ||
899 | doc.creationInfo.creators.append("Organization: %s" % d.getVar("SPDX_ORG")) | ||
900 | doc.creationInfo.creators.append("Person: N/A ()") | ||
901 | |||
902 | image = oe.spdx.SPDXPackage() | ||
903 | image.name = d.getVar("PN") | ||
904 | image.versionInfo = d.getVar("PV") | ||
905 | image.SPDXID = rootfs_spdxid | ||
906 | image.supplier = d.getVar("SPDX_SUPPLIER") | ||
907 | |||
908 | doc.packages.append(image) | ||
909 | |||
910 | for name in sorted(packages.keys()): | ||
911 | pkg_spdx_path = deploy_dir_spdx / "packages" / (name + ".spdx.json") | ||
912 | pkg_doc, pkg_doc_sha1 = oe.sbom.read_doc(pkg_spdx_path) | ||
913 | |||
914 | for p in pkg_doc.packages: | ||
915 | if p.name == name: | ||
916 | pkg_ref = oe.spdx.SPDXExternalDocumentRef() | ||
917 | pkg_ref.externalDocumentId = "DocumentRef-%s" % pkg_doc.name | ||
918 | pkg_ref.spdxDocument = pkg_doc.documentNamespace | ||
919 | pkg_ref.checksum.algorithm = "SHA1" | ||
920 | pkg_ref.checksum.checksumValue = pkg_doc_sha1 | ||
921 | |||
922 | doc.externalDocumentRefs.append(pkg_ref) | ||
923 | doc.add_relationship(image, "CONTAINS", "%s:%s" % (pkg_ref.externalDocumentId, p.SPDXID)) | ||
924 | break | ||
925 | else: | ||
926 | bb.fatal("Unable to find package with name '%s' in SPDX file %s" % (name, pkg_spdx_path)) | ||
927 | |||
928 | runtime_spdx_path = deploy_dir_spdx / "runtime" / ("runtime-" + name + ".spdx.json") | ||
929 | runtime_doc, runtime_doc_sha1 = oe.sbom.read_doc(runtime_spdx_path) | ||
930 | |||
931 | runtime_ref = oe.spdx.SPDXExternalDocumentRef() | ||
932 | runtime_ref.externalDocumentId = "DocumentRef-%s" % runtime_doc.name | ||
933 | runtime_ref.spdxDocument = runtime_doc.documentNamespace | ||
934 | runtime_ref.checksum.algorithm = "SHA1" | ||
935 | runtime_ref.checksum.checksumValue = runtime_doc_sha1 | ||
936 | |||
937 | # "OTHER" isn't ideal here, but I can't find a relationship that makes sense | ||
938 | doc.externalDocumentRefs.append(runtime_ref) | ||
939 | doc.add_relationship( | ||
940 | image, | ||
941 | "OTHER", | ||
942 | "%s:%s" % (runtime_ref.externalDocumentId, runtime_doc.SPDXID), | ||
943 | comment="Runtime dependencies for %s" % name | ||
944 | ) | ||
945 | |||
946 | image_spdx_path = rootfs_deploydir / (rootfs_name + ".spdx.json") | ||
947 | |||
948 | with image_spdx_path.open("wb") as f: | ||
949 | doc.to_json(f, sort_keys=True, indent=get_json_indent(d)) | ||
950 | |||
951 | num_threads = int(d.getVar("BB_NUMBER_THREADS")) | ||
952 | |||
953 | visited_docs = set() | ||
954 | |||
955 | index = {"documents": []} | ||
956 | |||
957 | spdx_tar_path = rootfs_deploydir / (rootfs_name + ".spdx.tar.zst") | ||
958 | with bb.compress.zstd.open(spdx_tar_path, "w", num_threads=num_threads) as f: | ||
959 | with tarfile.open(fileobj=f, mode="w|") as tar: | ||
960 | def collect_spdx_document(path): | ||
961 | nonlocal tar | ||
962 | nonlocal deploy_dir_spdx | ||
963 | nonlocal source_date_epoch | ||
964 | nonlocal index | ||
965 | |||
966 | if path in visited_docs: | ||
967 | return | ||
968 | |||
969 | visited_docs.add(path) | ||
970 | |||
971 | with path.open("rb") as f: | ||
972 | doc, sha1 = oe.sbom.read_doc(f) | ||
973 | f.seek(0) | ||
974 | |||
975 | if doc.documentNamespace in visited_docs: | ||
976 | return | ||
977 | |||
978 | bb.note("Adding SPDX document %s" % path) | ||
979 | visited_docs.add(doc.documentNamespace) | ||
980 | info = tar.gettarinfo(fileobj=f) | ||
981 | |||
982 | info.name = doc.name + ".spdx.json" | ||
983 | info.uid = 0 | ||
984 | info.gid = 0 | ||
985 | info.uname = "root" | ||
986 | info.gname = "root" | ||
987 | |||
988 | if source_date_epoch is not None and info.mtime > int(source_date_epoch): | ||
989 | info.mtime = int(source_date_epoch) | ||
990 | |||
991 | tar.addfile(info, f) | ||
992 | |||
993 | index["documents"].append({ | ||
994 | "filename": info.name, | ||
995 | "documentNamespace": doc.documentNamespace, | ||
996 | "sha1": sha1, | ||
997 | }) | ||
998 | |||
999 | for ref in doc.externalDocumentRefs: | ||
1000 | ref_path = deploy_dir_spdx / "by-namespace" / ref.spdxDocument.replace("/", "_") | ||
1001 | collect_spdx_document(ref_path) | ||
1002 | |||
1003 | collect_spdx_document(image_spdx_path) | ||
1004 | |||
1005 | index["documents"].sort(key=lambda x: x["filename"]) | ||
1006 | |||
1007 | index_str = io.BytesIO(json.dumps( | ||
1008 | index, | ||
1009 | sort_keys=True, | ||
1010 | indent=get_json_indent(d), | ||
1011 | ).encode("utf-8")) | ||
1012 | |||
1013 | info = tarfile.TarInfo() | ||
1014 | info.name = "index.json" | ||
1015 | info.size = len(index_str.getvalue()) | ||
1016 | info.uid = 0 | ||
1017 | info.gid = 0 | ||
1018 | info.uname = "root" | ||
1019 | info.gname = "root" | ||
1020 | |||
1021 | tar.addfile(info, fileobj=index_str) | ||
1022 | |||
1023 | spdx_index_path = rootfs_deploydir / (rootfs_name + ".spdx.index.json") | ||
1024 | with spdx_index_path.open("w") as f: | ||
1025 | json.dump(index, f, sort_keys=True, indent=get_json_indent(d)) | ||