diff options
Diffstat (limited to 'meta/lib/oe/packagedata.py')
-rw-r--r-- | meta/lib/oe/packagedata.py | 279 |
1 files changed, 275 insertions, 4 deletions
diff --git a/meta/lib/oe/packagedata.py b/meta/lib/oe/packagedata.py index a82085a792..b6a10a930a 100644 --- a/meta/lib/oe/packagedata.py +++ b/meta/lib/oe/packagedata.py | |||
@@ -1,9 +1,17 @@ | |||
1 | # | 1 | # |
2 | # Copyright OpenEmbedded Contributors | ||
3 | # | ||
2 | # SPDX-License-Identifier: GPL-2.0-only | 4 | # SPDX-License-Identifier: GPL-2.0-only |
3 | # | 5 | # |
4 | 6 | ||
5 | import codecs | 7 | import codecs |
6 | import os | 8 | import os |
9 | import json | ||
10 | import bb.parse | ||
11 | import bb.compress.zstd | ||
12 | import oe.path | ||
13 | |||
14 | from glob import glob | ||
7 | 15 | ||
8 | def packaged(pkg, d): | 16 | def packaged(pkg, d): |
9 | return os.access(get_subpkgedata_fn(pkg, d) + '.packaged', os.R_OK) | 17 | return os.access(get_subpkgedata_fn(pkg, d) + '.packaged', os.R_OK) |
@@ -19,7 +27,7 @@ def read_pkgdatafile(fn): | |||
19 | import re | 27 | import re |
20 | with open(fn, 'r') as f: | 28 | with open(fn, 'r') as f: |
21 | lines = f.readlines() | 29 | lines = f.readlines() |
22 | r = re.compile("([^:]+):\s*(.*)") | 30 | r = re.compile(r"(^.+?):\s+(.*)") |
23 | for l in lines: | 31 | for l in lines: |
24 | m = r.match(l) | 32 | m = r.match(l) |
25 | if m: | 33 | if m: |
@@ -45,18 +53,31 @@ def read_pkgdata(pn, d): | |||
45 | return read_pkgdatafile(fn) | 53 | return read_pkgdatafile(fn) |
46 | 54 | ||
47 | # | 55 | # |
48 | # Collapse FOO_pkg variables into FOO | 56 | # Collapse FOO:pkg variables into FOO |
49 | # | 57 | # |
50 | def read_subpkgdata_dict(pkg, d): | 58 | def read_subpkgdata_dict(pkg, d): |
51 | ret = {} | 59 | ret = {} |
52 | subd = read_pkgdatafile(get_subpkgedata_fn(pkg, d)) | 60 | subd = read_pkgdatafile(get_subpkgedata_fn(pkg, d)) |
53 | for var in subd: | 61 | for var in subd: |
54 | newvar = var.replace("_" + pkg, "") | 62 | newvar = var.replace(":" + pkg, "") |
55 | if newvar == var and var + "_" + pkg in subd: | 63 | if newvar == var and var + ":" + pkg in subd: |
56 | continue | 64 | continue |
57 | ret[newvar] = subd[var] | 65 | ret[newvar] = subd[var] |
58 | return ret | 66 | return ret |
59 | 67 | ||
68 | @bb.parse.vardepsexclude("BB_NUMBER_THREADS") | ||
69 | def read_subpkgdata_extended(pkg, d): | ||
70 | import json | ||
71 | import bb.compress.zstd | ||
72 | |||
73 | fn = d.expand("${PKGDATA_DIR}/extended/%s.json.zstd" % pkg) | ||
74 | try: | ||
75 | num_threads = int(d.getVar("BB_NUMBER_THREADS")) | ||
76 | with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f: | ||
77 | return json.load(f) | ||
78 | except FileNotFoundError: | ||
79 | return None | ||
80 | |||
60 | def _pkgmap(d): | 81 | def _pkgmap(d): |
61 | """Return a dictionary mapping package to recipe name.""" | 82 | """Return a dictionary mapping package to recipe name.""" |
62 | 83 | ||
@@ -96,3 +117,253 @@ def recipename(pkg, d): | |||
96 | """Return the recipe name for the given binary package name.""" | 117 | """Return the recipe name for the given binary package name.""" |
97 | 118 | ||
98 | return pkgmap(d).get(pkg) | 119 | return pkgmap(d).get(pkg) |
120 | |||
121 | def foreach_runtime_provider_pkgdata(d, rdep, include_rdep=False): | ||
122 | pkgdata_dir = d.getVar("PKGDATA_DIR") | ||
123 | possibles = set() | ||
124 | try: | ||
125 | possibles |= set(os.listdir("%s/runtime-rprovides/%s/" % (pkgdata_dir, rdep))) | ||
126 | except OSError: | ||
127 | pass | ||
128 | |||
129 | if include_rdep: | ||
130 | possibles.add(rdep) | ||
131 | |||
132 | for p in sorted(list(possibles)): | ||
133 | rdep_data = read_subpkgdata(p, d) | ||
134 | yield p, rdep_data | ||
135 | |||
136 | def get_package_mapping(pkg, basepkg, d, depversions=None): | ||
137 | import oe.packagedata | ||
138 | |||
139 | data = oe.packagedata.read_subpkgdata(pkg, d) | ||
140 | key = "PKG:%s" % pkg | ||
141 | |||
142 | if key in data: | ||
143 | if bb.data.inherits_class('allarch', d) and bb.data.inherits_class('packagegroup', d) and pkg != data[key]: | ||
144 | bb.error("An allarch packagegroup shouldn't depend on packages which are dynamically renamed (%s to %s)" % (pkg, data[key])) | ||
145 | # Have to avoid undoing the write_extra_pkgs(global_variants...) | ||
146 | if bb.data.inherits_class('allarch', d) and not d.getVar('MULTILIB_VARIANTS') \ | ||
147 | and data[key] == basepkg: | ||
148 | return pkg | ||
149 | if depversions == []: | ||
150 | # Avoid returning a mapping if the renamed package rprovides its original name | ||
151 | rprovkey = "RPROVIDES:%s" % pkg | ||
152 | if rprovkey in data: | ||
153 | if pkg in bb.utils.explode_dep_versions2(data[rprovkey]): | ||
154 | bb.note("%s rprovides %s, not replacing the latter" % (data[key], pkg)) | ||
155 | return pkg | ||
156 | # Do map to rewritten package name | ||
157 | return data[key] | ||
158 | |||
159 | return pkg | ||
160 | |||
161 | def get_package_additional_metadata(pkg_type, d): | ||
162 | base_key = "PACKAGE_ADD_METADATA" | ||
163 | for key in ("%s_%s" % (base_key, pkg_type.upper()), base_key): | ||
164 | if d.getVar(key, False) is None: | ||
165 | continue | ||
166 | d.setVarFlag(key, "type", "list") | ||
167 | if d.getVarFlag(key, "separator") is None: | ||
168 | d.setVarFlag(key, "separator", "\\n") | ||
169 | metadata_fields = [field.strip() for field in oe.data.typed_value(key, d)] | ||
170 | return "\n".join(metadata_fields).strip() | ||
171 | |||
172 | def runtime_mapping_rename(varname, pkg, d): | ||
173 | #bb.note("%s before: %s" % (varname, d.getVar(varname))) | ||
174 | |||
175 | new_depends = {} | ||
176 | deps = bb.utils.explode_dep_versions2(d.getVar(varname) or "") | ||
177 | for depend, depversions in deps.items(): | ||
178 | new_depend = get_package_mapping(depend, pkg, d, depversions) | ||
179 | if depend != new_depend: | ||
180 | bb.note("package name mapping done: %s -> %s" % (depend, new_depend)) | ||
181 | new_depends[new_depend] = deps[depend] | ||
182 | |||
183 | d.setVar(varname, bb.utils.join_deps(new_depends, commasep=False)) | ||
184 | |||
185 | #bb.note("%s after: %s" % (varname, d.getVar(varname))) | ||
186 | |||
187 | @bb.parse.vardepsexclude("BB_NUMBER_THREADS") | ||
188 | def emit_pkgdata(pkgfiles, d): | ||
189 | def process_postinst_on_target(pkg, mlprefix): | ||
190 | pkgval = d.getVar('PKG:%s' % pkg) | ||
191 | if pkgval is None: | ||
192 | pkgval = pkg | ||
193 | |||
194 | defer_fragment = """ | ||
195 | if [ -n "$D" ]; then | ||
196 | $INTERCEPT_DIR/postinst_intercept delay_to_first_boot %s mlprefix=%s | ||
197 | exit 0 | ||
198 | fi | ||
199 | """ % (pkgval, mlprefix) | ||
200 | |||
201 | postinst = d.getVar('pkg_postinst:%s' % pkg) | ||
202 | postinst_ontarget = d.getVar('pkg_postinst_ontarget:%s' % pkg) | ||
203 | |||
204 | if postinst_ontarget: | ||
205 | bb.debug(1, 'adding deferred pkg_postinst_ontarget() to pkg_postinst() for %s' % pkg) | ||
206 | if not postinst: | ||
207 | postinst = '#!/bin/sh\n' | ||
208 | postinst += defer_fragment | ||
209 | postinst += postinst_ontarget | ||
210 | d.setVar('pkg_postinst:%s' % pkg, postinst) | ||
211 | |||
212 | def add_set_e_to_scriptlets(pkg): | ||
213 | for scriptlet_name in ('pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'): | ||
214 | scriptlet = d.getVar('%s:%s' % (scriptlet_name, pkg)) | ||
215 | if scriptlet: | ||
216 | scriptlet_split = scriptlet.split('\n') | ||
217 | if scriptlet_split[0].startswith("#!"): | ||
218 | scriptlet = scriptlet_split[0] + "\nset -e\n" + "\n".join(scriptlet_split[1:]) | ||
219 | else: | ||
220 | scriptlet = "set -e\n" + "\n".join(scriptlet_split[0:]) | ||
221 | d.setVar('%s:%s' % (scriptlet_name, pkg), scriptlet) | ||
222 | |||
223 | def write_if_exists(f, pkg, var): | ||
224 | def encode(str): | ||
225 | import codecs | ||
226 | c = codecs.getencoder("unicode_escape") | ||
227 | return c(str)[0].decode("latin1") | ||
228 | |||
229 | val = d.getVar('%s:%s' % (var, pkg)) | ||
230 | if val: | ||
231 | f.write('%s:%s: %s\n' % (var, pkg, encode(val))) | ||
232 | return val | ||
233 | val = d.getVar('%s' % (var)) | ||
234 | if val: | ||
235 | f.write('%s: %s\n' % (var, encode(val))) | ||
236 | return val | ||
237 | |||
238 | def write_extra_pkgs(variants, pn, packages, pkgdatadir): | ||
239 | for variant in variants: | ||
240 | with open("%s/%s-%s" % (pkgdatadir, variant, pn), 'w') as fd: | ||
241 | fd.write("PACKAGES: %s\n" % ' '.join( | ||
242 | map(lambda pkg: '%s-%s' % (variant, pkg), packages.split()))) | ||
243 | |||
244 | def write_extra_runtime_pkgs(variants, packages, pkgdatadir): | ||
245 | for variant in variants: | ||
246 | for pkg in packages.split(): | ||
247 | ml_pkg = "%s-%s" % (variant, pkg) | ||
248 | subdata_file = "%s/runtime/%s" % (pkgdatadir, ml_pkg) | ||
249 | with open(subdata_file, 'w') as fd: | ||
250 | fd.write("PKG:%s: %s" % (ml_pkg, pkg)) | ||
251 | |||
252 | packages = d.getVar('PACKAGES') | ||
253 | pkgdest = d.getVar('PKGDEST') | ||
254 | pkgdatadir = d.getVar('PKGDESTWORK') | ||
255 | |||
256 | data_file = pkgdatadir + d.expand("/${PN}") | ||
257 | with open(data_file, 'w') as fd: | ||
258 | fd.write("PACKAGES: %s\n" % packages) | ||
259 | |||
260 | pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or [] | ||
261 | |||
262 | pn = d.getVar('PN') | ||
263 | global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() | ||
264 | variants = (d.getVar('MULTILIB_VARIANTS') or "").split() | ||
265 | |||
266 | if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d): | ||
267 | write_extra_pkgs(variants, pn, packages, pkgdatadir) | ||
268 | |||
269 | if bb.data.inherits_class('allarch', d) and not variants \ | ||
270 | and not bb.data.inherits_class('packagegroup', d): | ||
271 | write_extra_pkgs(global_variants, pn, packages, pkgdatadir) | ||
272 | |||
273 | workdir = d.getVar('WORKDIR') | ||
274 | |||
275 | for pkg in packages.split(): | ||
276 | pkgval = d.getVar('PKG:%s' % pkg) | ||
277 | if pkgval is None: | ||
278 | pkgval = pkg | ||
279 | d.setVar('PKG:%s' % pkg, pkg) | ||
280 | |||
281 | extended_data = { | ||
282 | "files_info": {} | ||
283 | } | ||
284 | |||
285 | pkgdestpkg = os.path.join(pkgdest, pkg) | ||
286 | files = {} | ||
287 | files_extra = {} | ||
288 | total_size = 0 | ||
289 | seen = set() | ||
290 | for f in pkgfiles[pkg]: | ||
291 | fpath = os.sep + os.path.relpath(f, pkgdestpkg) | ||
292 | |||
293 | fstat = os.lstat(f) | ||
294 | files[fpath] = fstat.st_size | ||
295 | |||
296 | extended_data["files_info"].setdefault(fpath, {}) | ||
297 | extended_data["files_info"][fpath]['size'] = fstat.st_size | ||
298 | |||
299 | if fstat.st_ino not in seen: | ||
300 | seen.add(fstat.st_ino) | ||
301 | total_size += fstat.st_size | ||
302 | |||
303 | if fpath in pkgdebugsource: | ||
304 | extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath] | ||
305 | del pkgdebugsource[fpath] | ||
306 | |||
307 | d.setVar('FILES_INFO:' + pkg , json.dumps(files, sort_keys=True)) | ||
308 | |||
309 | process_postinst_on_target(pkg, d.getVar("MLPREFIX")) | ||
310 | add_set_e_to_scriptlets(pkg) | ||
311 | |||
312 | subdata_file = pkgdatadir + "/runtime/%s" % pkg | ||
313 | with open(subdata_file, 'w') as sf: | ||
314 | for var in (d.getVar('PKGDATA_VARS') or "").split(): | ||
315 | val = write_if_exists(sf, pkg, var) | ||
316 | |||
317 | write_if_exists(sf, pkg, 'FILERPROVIDESFLIST') | ||
318 | for dfile in sorted((d.getVar('FILERPROVIDESFLIST:' + pkg) or "").split()): | ||
319 | write_if_exists(sf, pkg, 'FILERPROVIDES:' + dfile) | ||
320 | |||
321 | write_if_exists(sf, pkg, 'FILERDEPENDSFLIST') | ||
322 | for dfile in sorted((d.getVar('FILERDEPENDSFLIST:' + pkg) or "").split()): | ||
323 | write_if_exists(sf, pkg, 'FILERDEPENDS:' + dfile) | ||
324 | |||
325 | sf.write('%s:%s: %d\n' % ('PKGSIZE', pkg, total_size)) | ||
326 | |||
327 | subdata_extended_file = pkgdatadir + "/extended/%s.json.zstd" % pkg | ||
328 | num_threads = int(d.getVar("BB_NUMBER_THREADS")) | ||
329 | with bb.compress.zstd.open(subdata_extended_file, "wt", encoding="utf-8", num_threads=num_threads) as f: | ||
330 | json.dump(extended_data, f, sort_keys=True, separators=(",", ":")) | ||
331 | |||
332 | # Symlinks needed for rprovides lookup | ||
333 | rprov = d.getVar('RPROVIDES:%s' % pkg) or d.getVar('RPROVIDES') | ||
334 | if rprov: | ||
335 | for p in bb.utils.explode_deps(rprov): | ||
336 | subdata_sym = pkgdatadir + "/runtime-rprovides/%s/%s" % (p, pkg) | ||
337 | bb.utils.mkdirhier(os.path.dirname(subdata_sym)) | ||
338 | oe.path.relsymlink(subdata_file, subdata_sym, True) | ||
339 | |||
340 | allow_empty = d.getVar('ALLOW_EMPTY:%s' % pkg) | ||
341 | if not allow_empty: | ||
342 | allow_empty = d.getVar('ALLOW_EMPTY') | ||
343 | root = "%s/%s" % (pkgdest, pkg) | ||
344 | os.chdir(root) | ||
345 | g = glob('*') | ||
346 | if g or allow_empty == "1": | ||
347 | # Symlinks needed for reverse lookups (from the final package name) | ||
348 | subdata_sym = pkgdatadir + "/runtime-reverse/%s" % pkgval | ||
349 | oe.path.relsymlink(subdata_file, subdata_sym, True) | ||
350 | |||
351 | packagedfile = pkgdatadir + '/runtime/%s.packaged' % pkg | ||
352 | open(packagedfile, 'w').close() | ||
353 | |||
354 | if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d): | ||
355 | write_extra_runtime_pkgs(variants, packages, pkgdatadir) | ||
356 | |||
357 | if bb.data.inherits_class('allarch', d) and not variants \ | ||
358 | and not bb.data.inherits_class('packagegroup', d): | ||
359 | write_extra_runtime_pkgs(global_variants, packages, pkgdatadir) | ||
360 | |||
361 | def mapping_rename_hook(d): | ||
362 | """ | ||
363 | Rewrite variables to account for package renaming in things | ||
364 | like debian.bbclass or manual PKG variable name changes | ||
365 | """ | ||
366 | pkg = d.getVar("PKG") | ||
367 | oe.packagedata.runtime_mapping_rename("RDEPENDS", pkg, d) | ||
368 | oe.packagedata.runtime_mapping_rename("RRECOMMENDS", pkg, d) | ||
369 | oe.packagedata.runtime_mapping_rename("RSUGGESTS", pkg, d) | ||