summaryrefslogtreecommitdiffstats
path: root/meta/lib/oe/packagedata.py
diff options
context:
space:
mode:
Diffstat (limited to 'meta/lib/oe/packagedata.py')
-rw-r--r--meta/lib/oe/packagedata.py279
1 files changed, 275 insertions, 4 deletions
diff --git a/meta/lib/oe/packagedata.py b/meta/lib/oe/packagedata.py
index a82085a792..b6a10a930a 100644
--- a/meta/lib/oe/packagedata.py
+++ b/meta/lib/oe/packagedata.py
@@ -1,9 +1,17 @@
1# 1#
2# Copyright OpenEmbedded Contributors
3#
2# SPDX-License-Identifier: GPL-2.0-only 4# SPDX-License-Identifier: GPL-2.0-only
3# 5#
4 6
5import codecs 7import codecs
6import os 8import os
9import json
10import bb.parse
11import bb.compress.zstd
12import oe.path
13
14from glob import glob
7 15
8def packaged(pkg, d): 16def packaged(pkg, d):
9 return os.access(get_subpkgedata_fn(pkg, d) + '.packaged', os.R_OK) 17 return os.access(get_subpkgedata_fn(pkg, d) + '.packaged', os.R_OK)
@@ -19,7 +27,7 @@ def read_pkgdatafile(fn):
19 import re 27 import re
20 with open(fn, 'r') as f: 28 with open(fn, 'r') as f:
21 lines = f.readlines() 29 lines = f.readlines()
22 r = re.compile("([^:]+):\s*(.*)") 30 r = re.compile(r"(^.+?):\s+(.*)")
23 for l in lines: 31 for l in lines:
24 m = r.match(l) 32 m = r.match(l)
25 if m: 33 if m:
@@ -45,18 +53,31 @@ def read_pkgdata(pn, d):
45 return read_pkgdatafile(fn) 53 return read_pkgdatafile(fn)
46 54
47# 55#
48# Collapse FOO_pkg variables into FOO 56# Collapse FOO:pkg variables into FOO
49# 57#
50def read_subpkgdata_dict(pkg, d): 58def read_subpkgdata_dict(pkg, d):
51 ret = {} 59 ret = {}
52 subd = read_pkgdatafile(get_subpkgedata_fn(pkg, d)) 60 subd = read_pkgdatafile(get_subpkgedata_fn(pkg, d))
53 for var in subd: 61 for var in subd:
54 newvar = var.replace("_" + pkg, "") 62 newvar = var.replace(":" + pkg, "")
55 if newvar == var and var + "_" + pkg in subd: 63 if newvar == var and var + ":" + pkg in subd:
56 continue 64 continue
57 ret[newvar] = subd[var] 65 ret[newvar] = subd[var]
58 return ret 66 return ret
59 67
68@bb.parse.vardepsexclude("BB_NUMBER_THREADS")
69def read_subpkgdata_extended(pkg, d):
70 import json
71 import bb.compress.zstd
72
73 fn = d.expand("${PKGDATA_DIR}/extended/%s.json.zstd" % pkg)
74 try:
75 num_threads = int(d.getVar("BB_NUMBER_THREADS"))
76 with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f:
77 return json.load(f)
78 except FileNotFoundError:
79 return None
80
60def _pkgmap(d): 81def _pkgmap(d):
61 """Return a dictionary mapping package to recipe name.""" 82 """Return a dictionary mapping package to recipe name."""
62 83
@@ -96,3 +117,253 @@ def recipename(pkg, d):
96 """Return the recipe name for the given binary package name.""" 117 """Return the recipe name for the given binary package name."""
97 118
98 return pkgmap(d).get(pkg) 119 return pkgmap(d).get(pkg)
120
121def foreach_runtime_provider_pkgdata(d, rdep, include_rdep=False):
122 pkgdata_dir = d.getVar("PKGDATA_DIR")
123 possibles = set()
124 try:
125 possibles |= set(os.listdir("%s/runtime-rprovides/%s/" % (pkgdata_dir, rdep)))
126 except OSError:
127 pass
128
129 if include_rdep:
130 possibles.add(rdep)
131
132 for p in sorted(list(possibles)):
133 rdep_data = read_subpkgdata(p, d)
134 yield p, rdep_data
135
136def get_package_mapping(pkg, basepkg, d, depversions=None):
137 import oe.packagedata
138
139 data = oe.packagedata.read_subpkgdata(pkg, d)
140 key = "PKG:%s" % pkg
141
142 if key in data:
143 if bb.data.inherits_class('allarch', d) and bb.data.inherits_class('packagegroup', d) and pkg != data[key]:
144 bb.error("An allarch packagegroup shouldn't depend on packages which are dynamically renamed (%s to %s)" % (pkg, data[key]))
145 # Have to avoid undoing the write_extra_pkgs(global_variants...)
146 if bb.data.inherits_class('allarch', d) and not d.getVar('MULTILIB_VARIANTS') \
147 and data[key] == basepkg:
148 return pkg
149 if depversions == []:
150 # Avoid returning a mapping if the renamed package rprovides its original name
151 rprovkey = "RPROVIDES:%s" % pkg
152 if rprovkey in data:
153 if pkg in bb.utils.explode_dep_versions2(data[rprovkey]):
154 bb.note("%s rprovides %s, not replacing the latter" % (data[key], pkg))
155 return pkg
156 # Do map to rewritten package name
157 return data[key]
158
159 return pkg
160
161def get_package_additional_metadata(pkg_type, d):
162 base_key = "PACKAGE_ADD_METADATA"
163 for key in ("%s_%s" % (base_key, pkg_type.upper()), base_key):
164 if d.getVar(key, False) is None:
165 continue
166 d.setVarFlag(key, "type", "list")
167 if d.getVarFlag(key, "separator") is None:
168 d.setVarFlag(key, "separator", "\\n")
169 metadata_fields = [field.strip() for field in oe.data.typed_value(key, d)]
170 return "\n".join(metadata_fields).strip()
171
172def runtime_mapping_rename(varname, pkg, d):
173 #bb.note("%s before: %s" % (varname, d.getVar(varname)))
174
175 new_depends = {}
176 deps = bb.utils.explode_dep_versions2(d.getVar(varname) or "")
177 for depend, depversions in deps.items():
178 new_depend = get_package_mapping(depend, pkg, d, depversions)
179 if depend != new_depend:
180 bb.note("package name mapping done: %s -> %s" % (depend, new_depend))
181 new_depends[new_depend] = deps[depend]
182
183 d.setVar(varname, bb.utils.join_deps(new_depends, commasep=False))
184
185 #bb.note("%s after: %s" % (varname, d.getVar(varname)))
186
187@bb.parse.vardepsexclude("BB_NUMBER_THREADS")
188def emit_pkgdata(pkgfiles, d):
189 def process_postinst_on_target(pkg, mlprefix):
190 pkgval = d.getVar('PKG:%s' % pkg)
191 if pkgval is None:
192 pkgval = pkg
193
194 defer_fragment = """
195if [ -n "$D" ]; then
196 $INTERCEPT_DIR/postinst_intercept delay_to_first_boot %s mlprefix=%s
197 exit 0
198fi
199""" % (pkgval, mlprefix)
200
201 postinst = d.getVar('pkg_postinst:%s' % pkg)
202 postinst_ontarget = d.getVar('pkg_postinst_ontarget:%s' % pkg)
203
204 if postinst_ontarget:
205 bb.debug(1, 'adding deferred pkg_postinst_ontarget() to pkg_postinst() for %s' % pkg)
206 if not postinst:
207 postinst = '#!/bin/sh\n'
208 postinst += defer_fragment
209 postinst += postinst_ontarget
210 d.setVar('pkg_postinst:%s' % pkg, postinst)
211
212 def add_set_e_to_scriptlets(pkg):
213 for scriptlet_name in ('pkg_preinst', 'pkg_postinst', 'pkg_prerm', 'pkg_postrm'):
214 scriptlet = d.getVar('%s:%s' % (scriptlet_name, pkg))
215 if scriptlet:
216 scriptlet_split = scriptlet.split('\n')
217 if scriptlet_split[0].startswith("#!"):
218 scriptlet = scriptlet_split[0] + "\nset -e\n" + "\n".join(scriptlet_split[1:])
219 else:
220 scriptlet = "set -e\n" + "\n".join(scriptlet_split[0:])
221 d.setVar('%s:%s' % (scriptlet_name, pkg), scriptlet)
222
223 def write_if_exists(f, pkg, var):
224 def encode(str):
225 import codecs
226 c = codecs.getencoder("unicode_escape")
227 return c(str)[0].decode("latin1")
228
229 val = d.getVar('%s:%s' % (var, pkg))
230 if val:
231 f.write('%s:%s: %s\n' % (var, pkg, encode(val)))
232 return val
233 val = d.getVar('%s' % (var))
234 if val:
235 f.write('%s: %s\n' % (var, encode(val)))
236 return val
237
238 def write_extra_pkgs(variants, pn, packages, pkgdatadir):
239 for variant in variants:
240 with open("%s/%s-%s" % (pkgdatadir, variant, pn), 'w') as fd:
241 fd.write("PACKAGES: %s\n" % ' '.join(
242 map(lambda pkg: '%s-%s' % (variant, pkg), packages.split())))
243
244 def write_extra_runtime_pkgs(variants, packages, pkgdatadir):
245 for variant in variants:
246 for pkg in packages.split():
247 ml_pkg = "%s-%s" % (variant, pkg)
248 subdata_file = "%s/runtime/%s" % (pkgdatadir, ml_pkg)
249 with open(subdata_file, 'w') as fd:
250 fd.write("PKG:%s: %s" % (ml_pkg, pkg))
251
252 packages = d.getVar('PACKAGES')
253 pkgdest = d.getVar('PKGDEST')
254 pkgdatadir = d.getVar('PKGDESTWORK')
255
256 data_file = pkgdatadir + d.expand("/${PN}")
257 with open(data_file, 'w') as fd:
258 fd.write("PACKAGES: %s\n" % packages)
259
260 pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or []
261
262 pn = d.getVar('PN')
263 global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split()
264 variants = (d.getVar('MULTILIB_VARIANTS') or "").split()
265
266 if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d):
267 write_extra_pkgs(variants, pn, packages, pkgdatadir)
268
269 if bb.data.inherits_class('allarch', d) and not variants \
270 and not bb.data.inherits_class('packagegroup', d):
271 write_extra_pkgs(global_variants, pn, packages, pkgdatadir)
272
273 workdir = d.getVar('WORKDIR')
274
275 for pkg in packages.split():
276 pkgval = d.getVar('PKG:%s' % pkg)
277 if pkgval is None:
278 pkgval = pkg
279 d.setVar('PKG:%s' % pkg, pkg)
280
281 extended_data = {
282 "files_info": {}
283 }
284
285 pkgdestpkg = os.path.join(pkgdest, pkg)
286 files = {}
287 files_extra = {}
288 total_size = 0
289 seen = set()
290 for f in pkgfiles[pkg]:
291 fpath = os.sep + os.path.relpath(f, pkgdestpkg)
292
293 fstat = os.lstat(f)
294 files[fpath] = fstat.st_size
295
296 extended_data["files_info"].setdefault(fpath, {})
297 extended_data["files_info"][fpath]['size'] = fstat.st_size
298
299 if fstat.st_ino not in seen:
300 seen.add(fstat.st_ino)
301 total_size += fstat.st_size
302
303 if fpath in pkgdebugsource:
304 extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath]
305 del pkgdebugsource[fpath]
306
307 d.setVar('FILES_INFO:' + pkg , json.dumps(files, sort_keys=True))
308
309 process_postinst_on_target(pkg, d.getVar("MLPREFIX"))
310 add_set_e_to_scriptlets(pkg)
311
312 subdata_file = pkgdatadir + "/runtime/%s" % pkg
313 with open(subdata_file, 'w') as sf:
314 for var in (d.getVar('PKGDATA_VARS') or "").split():
315 val = write_if_exists(sf, pkg, var)
316
317 write_if_exists(sf, pkg, 'FILERPROVIDESFLIST')
318 for dfile in sorted((d.getVar('FILERPROVIDESFLIST:' + pkg) or "").split()):
319 write_if_exists(sf, pkg, 'FILERPROVIDES:' + dfile)
320
321 write_if_exists(sf, pkg, 'FILERDEPENDSFLIST')
322 for dfile in sorted((d.getVar('FILERDEPENDSFLIST:' + pkg) or "").split()):
323 write_if_exists(sf, pkg, 'FILERDEPENDS:' + dfile)
324
325 sf.write('%s:%s: %d\n' % ('PKGSIZE', pkg, total_size))
326
327 subdata_extended_file = pkgdatadir + "/extended/%s.json.zstd" % pkg
328 num_threads = int(d.getVar("BB_NUMBER_THREADS"))
329 with bb.compress.zstd.open(subdata_extended_file, "wt", encoding="utf-8", num_threads=num_threads) as f:
330 json.dump(extended_data, f, sort_keys=True, separators=(",", ":"))
331
332 # Symlinks needed for rprovides lookup
333 rprov = d.getVar('RPROVIDES:%s' % pkg) or d.getVar('RPROVIDES')
334 if rprov:
335 for p in bb.utils.explode_deps(rprov):
336 subdata_sym = pkgdatadir + "/runtime-rprovides/%s/%s" % (p, pkg)
337 bb.utils.mkdirhier(os.path.dirname(subdata_sym))
338 oe.path.relsymlink(subdata_file, subdata_sym, True)
339
340 allow_empty = d.getVar('ALLOW_EMPTY:%s' % pkg)
341 if not allow_empty:
342 allow_empty = d.getVar('ALLOW_EMPTY')
343 root = "%s/%s" % (pkgdest, pkg)
344 os.chdir(root)
345 g = glob('*')
346 if g or allow_empty == "1":
347 # Symlinks needed for reverse lookups (from the final package name)
348 subdata_sym = pkgdatadir + "/runtime-reverse/%s" % pkgval
349 oe.path.relsymlink(subdata_file, subdata_sym, True)
350
351 packagedfile = pkgdatadir + '/runtime/%s.packaged' % pkg
352 open(packagedfile, 'w').close()
353
354 if bb.data.inherits_class('kernel', d) or bb.data.inherits_class('module-base', d):
355 write_extra_runtime_pkgs(variants, packages, pkgdatadir)
356
357 if bb.data.inherits_class('allarch', d) and not variants \
358 and not bb.data.inherits_class('packagegroup', d):
359 write_extra_runtime_pkgs(global_variants, packages, pkgdatadir)
360
361def mapping_rename_hook(d):
362 """
363 Rewrite variables to account for package renaming in things
364 like debian.bbclass or manual PKG variable name changes
365 """
366 pkg = d.getVar("PKG")
367 oe.packagedata.runtime_mapping_rename("RDEPENDS", pkg, d)
368 oe.packagedata.runtime_mapping_rename("RRECOMMENDS", pkg, d)
369 oe.packagedata.runtime_mapping_rename("RSUGGESTS", pkg, d)