diff options
Diffstat (limited to 'meta/lib')
-rw-r--r-- | meta/lib/oe/sbom.py | 84 | ||||
-rw-r--r-- | meta/lib/oe/spdx.py | 357 |
2 files changed, 441 insertions, 0 deletions
diff --git a/meta/lib/oe/sbom.py b/meta/lib/oe/sbom.py new file mode 100644 index 0000000000..22ed5070ea --- /dev/null +++ b/meta/lib/oe/sbom.py | |||
@@ -0,0 +1,84 @@ | |||
1 | # | ||
2 | # Copyright OpenEmbedded Contributors | ||
3 | # | ||
4 | # SPDX-License-Identifier: GPL-2.0-only | ||
5 | # | ||
6 | |||
7 | import collections | ||
8 | |||
9 | DepRecipe = collections.namedtuple("DepRecipe", ("doc", "doc_sha1", "recipe")) | ||
10 | DepSource = collections.namedtuple("DepSource", ("doc", "doc_sha1", "recipe", "file")) | ||
11 | |||
12 | |||
13 | def get_recipe_spdxid(d): | ||
14 | return "SPDXRef-%s-%s" % ("Recipe", d.getVar("PN")) | ||
15 | |||
16 | |||
17 | def get_download_spdxid(d, idx): | ||
18 | return "SPDXRef-Download-%s-%d" % (d.getVar("PN"), idx) | ||
19 | |||
20 | |||
21 | def get_package_spdxid(pkg): | ||
22 | return "SPDXRef-Package-%s" % pkg | ||
23 | |||
24 | |||
25 | def get_source_file_spdxid(d, idx): | ||
26 | return "SPDXRef-SourceFile-%s-%d" % (d.getVar("PN"), idx) | ||
27 | |||
28 | |||
29 | def get_packaged_file_spdxid(pkg, idx): | ||
30 | return "SPDXRef-PackagedFile-%s-%d" % (pkg, idx) | ||
31 | |||
32 | |||
33 | def get_image_spdxid(img): | ||
34 | return "SPDXRef-Image-%s" % img | ||
35 | |||
36 | |||
37 | def get_sdk_spdxid(sdk): | ||
38 | return "SPDXRef-SDK-%s" % sdk | ||
39 | |||
40 | |||
41 | def write_doc(d, spdx_doc, subdir, spdx_deploy=None, indent=None): | ||
42 | from pathlib import Path | ||
43 | |||
44 | if spdx_deploy is None: | ||
45 | spdx_deploy = Path(d.getVar("SPDXDEPLOY")) | ||
46 | |||
47 | dest = spdx_deploy / subdir / (spdx_doc.name + ".spdx.json") | ||
48 | dest.parent.mkdir(exist_ok=True, parents=True) | ||
49 | with dest.open("wb") as f: | ||
50 | doc_sha1 = spdx_doc.to_json(f, sort_keys=True, indent=indent) | ||
51 | |||
52 | l = spdx_deploy / "by-namespace" / spdx_doc.documentNamespace.replace("/", "_") | ||
53 | l.parent.mkdir(exist_ok=True, parents=True) | ||
54 | l.symlink_to(os.path.relpath(dest, l.parent)) | ||
55 | |||
56 | return doc_sha1 | ||
57 | |||
58 | |||
59 | def read_doc(fn): | ||
60 | import hashlib | ||
61 | import oe.spdx | ||
62 | import io | ||
63 | import contextlib | ||
64 | |||
65 | @contextlib.contextmanager | ||
66 | def get_file(): | ||
67 | if isinstance(fn, io.IOBase): | ||
68 | yield fn | ||
69 | else: | ||
70 | with fn.open("rb") as f: | ||
71 | yield f | ||
72 | |||
73 | with get_file() as f: | ||
74 | sha1 = hashlib.sha1() | ||
75 | while True: | ||
76 | chunk = f.read(4096) | ||
77 | if not chunk: | ||
78 | break | ||
79 | sha1.update(chunk) | ||
80 | |||
81 | f.seek(0) | ||
82 | doc = oe.spdx.SPDXDocument.from_json(f) | ||
83 | |||
84 | return (doc, sha1.hexdigest()) | ||
diff --git a/meta/lib/oe/spdx.py b/meta/lib/oe/spdx.py new file mode 100644 index 0000000000..7aaf2af5ed --- /dev/null +++ b/meta/lib/oe/spdx.py | |||
@@ -0,0 +1,357 @@ | |||
1 | # | ||
2 | # Copyright OpenEmbedded Contributors | ||
3 | # | ||
4 | # SPDX-License-Identifier: GPL-2.0-only | ||
5 | # | ||
6 | |||
7 | # | ||
8 | # This library is intended to capture the JSON SPDX specification in a type | ||
9 | # safe manner. It is not intended to encode any particular OE specific | ||
10 | # behaviors, see the sbom.py for that. | ||
11 | # | ||
12 | # The documented SPDX spec document doesn't cover the JSON syntax for | ||
13 | # particular configuration, which can make it hard to determine what the JSON | ||
14 | # syntax should be. I've found it is actually much simpler to read the official | ||
15 | # SPDX JSON schema which can be found here: https://github.com/spdx/spdx-spec | ||
16 | # in schemas/spdx-schema.json | ||
17 | # | ||
18 | |||
19 | import hashlib | ||
20 | import itertools | ||
21 | import json | ||
22 | |||
23 | SPDX_VERSION = "2.2" | ||
24 | |||
25 | |||
26 | # | ||
27 | # The following are the support classes that are used to implement SPDX object | ||
28 | # | ||
29 | |||
30 | class _Property(object): | ||
31 | """ | ||
32 | A generic SPDX object property. The different types will derive from this | ||
33 | class | ||
34 | """ | ||
35 | |||
36 | def __init__(self, *, default=None): | ||
37 | self.default = default | ||
38 | |||
39 | def setdefault(self, dest, name): | ||
40 | if self.default is not None: | ||
41 | dest.setdefault(name, self.default) | ||
42 | |||
43 | |||
44 | class _String(_Property): | ||
45 | """ | ||
46 | A scalar string property for an SPDX object | ||
47 | """ | ||
48 | |||
49 | def __init__(self, **kwargs): | ||
50 | super().__init__(**kwargs) | ||
51 | |||
52 | def set_property(self, attrs, name): | ||
53 | def get_helper(obj): | ||
54 | return obj._spdx[name] | ||
55 | |||
56 | def set_helper(obj, value): | ||
57 | obj._spdx[name] = value | ||
58 | |||
59 | def del_helper(obj): | ||
60 | del obj._spdx[name] | ||
61 | |||
62 | attrs[name] = property(get_helper, set_helper, del_helper) | ||
63 | |||
64 | def init(self, source): | ||
65 | return source | ||
66 | |||
67 | |||
68 | class _Object(_Property): | ||
69 | """ | ||
70 | A scalar SPDX object property of a SPDX object | ||
71 | """ | ||
72 | |||
73 | def __init__(self, cls, **kwargs): | ||
74 | super().__init__(**kwargs) | ||
75 | self.cls = cls | ||
76 | |||
77 | def set_property(self, attrs, name): | ||
78 | def get_helper(obj): | ||
79 | if not name in obj._spdx: | ||
80 | obj._spdx[name] = self.cls() | ||
81 | return obj._spdx[name] | ||
82 | |||
83 | def set_helper(obj, value): | ||
84 | obj._spdx[name] = value | ||
85 | |||
86 | def del_helper(obj): | ||
87 | del obj._spdx[name] | ||
88 | |||
89 | attrs[name] = property(get_helper, set_helper) | ||
90 | |||
91 | def init(self, source): | ||
92 | return self.cls(**source) | ||
93 | |||
94 | |||
95 | class _ListProperty(_Property): | ||
96 | """ | ||
97 | A list of SPDX properties | ||
98 | """ | ||
99 | |||
100 | def __init__(self, prop, **kwargs): | ||
101 | super().__init__(**kwargs) | ||
102 | self.prop = prop | ||
103 | |||
104 | def set_property(self, attrs, name): | ||
105 | def get_helper(obj): | ||
106 | if not name in obj._spdx: | ||
107 | obj._spdx[name] = [] | ||
108 | return obj._spdx[name] | ||
109 | |||
110 | def set_helper(obj, value): | ||
111 | obj._spdx[name] = list(value) | ||
112 | |||
113 | def del_helper(obj): | ||
114 | del obj._spdx[name] | ||
115 | |||
116 | attrs[name] = property(get_helper, set_helper, del_helper) | ||
117 | |||
118 | def init(self, source): | ||
119 | return [self.prop.init(o) for o in source] | ||
120 | |||
121 | |||
122 | class _StringList(_ListProperty): | ||
123 | """ | ||
124 | A list of strings as a property for an SPDX object | ||
125 | """ | ||
126 | |||
127 | def __init__(self, **kwargs): | ||
128 | super().__init__(_String(), **kwargs) | ||
129 | |||
130 | |||
131 | class _ObjectList(_ListProperty): | ||
132 | """ | ||
133 | A list of SPDX objects as a property for an SPDX object | ||
134 | """ | ||
135 | |||
136 | def __init__(self, cls, **kwargs): | ||
137 | super().__init__(_Object(cls), **kwargs) | ||
138 | |||
139 | |||
140 | class MetaSPDXObject(type): | ||
141 | """ | ||
142 | A metaclass that allows properties (anything derived from a _Property | ||
143 | class) to be defined for a SPDX object | ||
144 | """ | ||
145 | def __new__(mcls, name, bases, attrs): | ||
146 | attrs["_properties"] = {} | ||
147 | |||
148 | for key in attrs.keys(): | ||
149 | if isinstance(attrs[key], _Property): | ||
150 | prop = attrs[key] | ||
151 | attrs["_properties"][key] = prop | ||
152 | prop.set_property(attrs, key) | ||
153 | |||
154 | return super().__new__(mcls, name, bases, attrs) | ||
155 | |||
156 | |||
157 | class SPDXObject(metaclass=MetaSPDXObject): | ||
158 | """ | ||
159 | The base SPDX object; all SPDX spec classes must derive from this class | ||
160 | """ | ||
161 | def __init__(self, **d): | ||
162 | self._spdx = {} | ||
163 | |||
164 | for name, prop in self._properties.items(): | ||
165 | prop.setdefault(self._spdx, name) | ||
166 | if name in d: | ||
167 | self._spdx[name] = prop.init(d[name]) | ||
168 | |||
169 | def serializer(self): | ||
170 | return self._spdx | ||
171 | |||
172 | def __setattr__(self, name, value): | ||
173 | if name in self._properties or name == "_spdx": | ||
174 | super().__setattr__(name, value) | ||
175 | return | ||
176 | raise KeyError("%r is not a valid SPDX property" % name) | ||
177 | |||
178 | # | ||
179 | # These are the SPDX objects implemented from the spec. The *only* properties | ||
180 | # that can be added to these objects are ones directly specified in the SPDX | ||
181 | # spec, however you may add helper functions to make operations easier. | ||
182 | # | ||
183 | # Defaults should *only* be specified if the SPDX spec says there is a certain | ||
184 | # required value for a field (e.g. dataLicense), or if the field is mandatory | ||
185 | # and has some sane "this field is unknown" (e.g. "NOASSERTION") | ||
186 | # | ||
187 | |||
188 | class SPDXAnnotation(SPDXObject): | ||
189 | annotationDate = _String() | ||
190 | annotationType = _String() | ||
191 | annotator = _String() | ||
192 | comment = _String() | ||
193 | |||
194 | class SPDXChecksum(SPDXObject): | ||
195 | algorithm = _String() | ||
196 | checksumValue = _String() | ||
197 | |||
198 | |||
199 | class SPDXRelationship(SPDXObject): | ||
200 | spdxElementId = _String() | ||
201 | relatedSpdxElement = _String() | ||
202 | relationshipType = _String() | ||
203 | comment = _String() | ||
204 | annotations = _ObjectList(SPDXAnnotation) | ||
205 | |||
206 | |||
207 | class SPDXExternalReference(SPDXObject): | ||
208 | referenceCategory = _String() | ||
209 | referenceType = _String() | ||
210 | referenceLocator = _String() | ||
211 | |||
212 | |||
213 | class SPDXPackageVerificationCode(SPDXObject): | ||
214 | packageVerificationCodeValue = _String() | ||
215 | packageVerificationCodeExcludedFiles = _StringList() | ||
216 | |||
217 | |||
218 | class SPDXPackage(SPDXObject): | ||
219 | ALLOWED_CHECKSUMS = [ | ||
220 | "SHA1", | ||
221 | "SHA224", | ||
222 | "SHA256", | ||
223 | "SHA384", | ||
224 | "SHA512", | ||
225 | "MD2", | ||
226 | "MD4", | ||
227 | "MD5", | ||
228 | "MD6", | ||
229 | ] | ||
230 | |||
231 | name = _String() | ||
232 | SPDXID = _String() | ||
233 | versionInfo = _String() | ||
234 | downloadLocation = _String(default="NOASSERTION") | ||
235 | supplier = _String(default="NOASSERTION") | ||
236 | homepage = _String() | ||
237 | licenseConcluded = _String(default="NOASSERTION") | ||
238 | licenseDeclared = _String(default="NOASSERTION") | ||
239 | summary = _String() | ||
240 | description = _String() | ||
241 | sourceInfo = _String() | ||
242 | copyrightText = _String(default="NOASSERTION") | ||
243 | licenseInfoFromFiles = _StringList(default=["NOASSERTION"]) | ||
244 | externalRefs = _ObjectList(SPDXExternalReference) | ||
245 | packageVerificationCode = _Object(SPDXPackageVerificationCode) | ||
246 | hasFiles = _StringList() | ||
247 | packageFileName = _String() | ||
248 | annotations = _ObjectList(SPDXAnnotation) | ||
249 | checksums = _ObjectList(SPDXChecksum) | ||
250 | |||
251 | |||
252 | class SPDXFile(SPDXObject): | ||
253 | SPDXID = _String() | ||
254 | fileName = _String() | ||
255 | licenseConcluded = _String(default="NOASSERTION") | ||
256 | copyrightText = _String(default="NOASSERTION") | ||
257 | licenseInfoInFiles = _StringList(default=["NOASSERTION"]) | ||
258 | checksums = _ObjectList(SPDXChecksum) | ||
259 | fileTypes = _StringList() | ||
260 | |||
261 | |||
262 | class SPDXCreationInfo(SPDXObject): | ||
263 | created = _String() | ||
264 | licenseListVersion = _String() | ||
265 | comment = _String() | ||
266 | creators = _StringList() | ||
267 | |||
268 | |||
269 | class SPDXExternalDocumentRef(SPDXObject): | ||
270 | externalDocumentId = _String() | ||
271 | spdxDocument = _String() | ||
272 | checksum = _Object(SPDXChecksum) | ||
273 | |||
274 | |||
275 | class SPDXExtractedLicensingInfo(SPDXObject): | ||
276 | name = _String() | ||
277 | comment = _String() | ||
278 | licenseId = _String() | ||
279 | extractedText = _String() | ||
280 | |||
281 | |||
282 | class SPDXDocument(SPDXObject): | ||
283 | spdxVersion = _String(default="SPDX-" + SPDX_VERSION) | ||
284 | dataLicense = _String(default="CC0-1.0") | ||
285 | SPDXID = _String(default="SPDXRef-DOCUMENT") | ||
286 | name = _String() | ||
287 | documentNamespace = _String() | ||
288 | creationInfo = _Object(SPDXCreationInfo) | ||
289 | packages = _ObjectList(SPDXPackage) | ||
290 | files = _ObjectList(SPDXFile) | ||
291 | relationships = _ObjectList(SPDXRelationship) | ||
292 | externalDocumentRefs = _ObjectList(SPDXExternalDocumentRef) | ||
293 | hasExtractedLicensingInfos = _ObjectList(SPDXExtractedLicensingInfo) | ||
294 | |||
295 | def __init__(self, **d): | ||
296 | super().__init__(**d) | ||
297 | |||
298 | def to_json(self, f, *, sort_keys=False, indent=None, separators=None): | ||
299 | class Encoder(json.JSONEncoder): | ||
300 | def default(self, o): | ||
301 | if isinstance(o, SPDXObject): | ||
302 | return o.serializer() | ||
303 | |||
304 | return super().default(o) | ||
305 | |||
306 | sha1 = hashlib.sha1() | ||
307 | for chunk in Encoder( | ||
308 | sort_keys=sort_keys, | ||
309 | indent=indent, | ||
310 | separators=separators, | ||
311 | ).iterencode(self): | ||
312 | chunk = chunk.encode("utf-8") | ||
313 | f.write(chunk) | ||
314 | sha1.update(chunk) | ||
315 | |||
316 | return sha1.hexdigest() | ||
317 | |||
318 | @classmethod | ||
319 | def from_json(cls, f): | ||
320 | return cls(**json.load(f)) | ||
321 | |||
322 | def add_relationship(self, _from, relationship, _to, *, comment=None, annotation=None): | ||
323 | if isinstance(_from, SPDXObject): | ||
324 | from_spdxid = _from.SPDXID | ||
325 | else: | ||
326 | from_spdxid = _from | ||
327 | |||
328 | if isinstance(_to, SPDXObject): | ||
329 | to_spdxid = _to.SPDXID | ||
330 | else: | ||
331 | to_spdxid = _to | ||
332 | |||
333 | r = SPDXRelationship( | ||
334 | spdxElementId=from_spdxid, | ||
335 | relatedSpdxElement=to_spdxid, | ||
336 | relationshipType=relationship, | ||
337 | ) | ||
338 | |||
339 | if comment is not None: | ||
340 | r.comment = comment | ||
341 | |||
342 | if annotation is not None: | ||
343 | r.annotations.append(annotation) | ||
344 | |||
345 | self.relationships.append(r) | ||
346 | |||
347 | def find_by_spdxid(self, spdxid): | ||
348 | for o in itertools.chain(self.packages, self.files): | ||
349 | if o.SPDXID == spdxid: | ||
350 | return o | ||
351 | return None | ||
352 | |||
353 | def find_external_document_ref(self, namespace): | ||
354 | for r in self.externalDocumentRefs: | ||
355 | if r.spdxDocument == namespace: | ||
356 | return r | ||
357 | return None | ||