diff options
author | Joshua Watt <JPEWhacker@gmail.com> | 2021-09-01 08:44:40 -0500 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2021-09-03 09:53:28 +0100 |
commit | 7ec54b174304e940ec66f21ac512f7b50fa637b3 (patch) | |
tree | b072a6f30ca0e48d5ddced075a58257b232347b4 | |
parent | d521ba8c32cc1028304369b754d9d9dbf0f0e60d (diff) | |
download | poky-7ec54b174304e940ec66f21ac512f7b50fa637b3.tar.gz |
classes/package: Add extended packaged data
Adds extended package data which is encoded as JSON which allows it to
encode more structure than the "flat" package data files. The extended
data might be much larger than the standard package data, so it is not
read by default and instead requires
oe.packagedata.read_subpkgdata_extended() to be called
Currently, the file sizes and ELF debug sources are saved off into the
extended package data
(From OE-Core rev: db9cf430e54ae68da80fbc3fba80ce88d8df164d)
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Reviewed-by: Saul Wold <saul.wold@windriver.com>
Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | meta/classes/package.bbclass | 40 | ||||
-rw-r--r-- | meta/lib/oe/packagedata.py | 12 |
2 files changed, 49 insertions, 3 deletions
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass index c4c5515d5d..3a78e48da4 100644 --- a/meta/classes/package.bbclass +++ b/meta/classes/package.bbclass | |||
@@ -1224,6 +1224,14 @@ python split_and_strip_files () { | |||
1224 | # Modified the file so clear the cache | 1224 | # Modified the file so clear the cache |
1225 | cpath.updatecache(file) | 1225 | cpath.updatecache(file) |
1226 | 1226 | ||
1227 | def strip_pkgd_prefix(f): | ||
1228 | nonlocal dvar | ||
1229 | |||
1230 | if f.startswith(dvar): | ||
1231 | return f[len(dvar):] | ||
1232 | |||
1233 | return f | ||
1234 | |||
1227 | # | 1235 | # |
1228 | # First lets process debug splitting | 1236 | # First lets process debug splitting |
1229 | # | 1237 | # |
@@ -1237,6 +1245,8 @@ python split_and_strip_files () { | |||
1237 | for file in staticlibs: | 1245 | for file in staticlibs: |
1238 | results.append( (file,source_info(file, d)) ) | 1246 | results.append( (file,source_info(file, d)) ) |
1239 | 1247 | ||
1248 | d.setVar("PKGDEBUGSOURCES", {strip_pkgd_prefix(f): sorted(s) for f, s in results}) | ||
1249 | |||
1240 | sources = set() | 1250 | sources = set() |
1241 | for r in results: | 1251 | for r in results: |
1242 | sources.update(r[1]) | 1252 | sources.update(r[1]) |
@@ -1549,6 +1559,7 @@ PKGDATA_VARS = "PN PE PV PR PKGE PKGV PKGR LICENSE DESCRIPTION SUMMARY RDEPENDS | |||
1549 | python emit_pkgdata() { | 1559 | python emit_pkgdata() { |
1550 | from glob import glob | 1560 | from glob import glob |
1551 | import json | 1561 | import json |
1562 | import bb.compress.zstd | ||
1552 | 1563 | ||
1553 | def process_postinst_on_target(pkg, mlprefix): | 1564 | def process_postinst_on_target(pkg, mlprefix): |
1554 | pkgval = d.getVar('PKG:%s' % pkg) | 1565 | pkgval = d.getVar('PKG:%s' % pkg) |
@@ -1621,6 +1632,8 @@ fi | |||
1621 | with open(data_file, 'w') as fd: | 1632 | with open(data_file, 'w') as fd: |
1622 | fd.write("PACKAGES: %s\n" % packages) | 1633 | fd.write("PACKAGES: %s\n" % packages) |
1623 | 1634 | ||
1635 | pkgdebugsource = d.getVar("PKGDEBUGSOURCES") or [] | ||
1636 | |||
1624 | pn = d.getVar('PN') | 1637 | pn = d.getVar('PN') |
1625 | global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() | 1638 | global_variants = (d.getVar('MULTILIB_GLOBAL_VARIANTS') or "").split() |
1626 | variants = (d.getVar('MULTILIB_VARIANTS') or "").split() | 1639 | variants = (d.getVar('MULTILIB_VARIANTS') or "").split() |
@@ -1640,17 +1653,32 @@ fi | |||
1640 | pkgval = pkg | 1653 | pkgval = pkg |
1641 | d.setVar('PKG:%s' % pkg, pkg) | 1654 | d.setVar('PKG:%s' % pkg, pkg) |
1642 | 1655 | ||
1656 | extended_data = { | ||
1657 | "files_info": {} | ||
1658 | } | ||
1659 | |||
1643 | pkgdestpkg = os.path.join(pkgdest, pkg) | 1660 | pkgdestpkg = os.path.join(pkgdest, pkg) |
1644 | files = {} | 1661 | files = {} |
1662 | files_extra = {} | ||
1645 | total_size = 0 | 1663 | total_size = 0 |
1646 | seen = set() | 1664 | seen = set() |
1647 | for f in pkgfiles[pkg]: | 1665 | for f in pkgfiles[pkg]: |
1648 | relpth = os.path.relpath(f, pkgdestpkg) | 1666 | fpath = os.sep + os.path.relpath(f, pkgdestpkg) |
1667 | |||
1649 | fstat = os.lstat(f) | 1668 | fstat = os.lstat(f) |
1650 | files[os.sep + relpth] = fstat.st_size | 1669 | files[fpath] = fstat.st_size |
1670 | |||
1671 | extended_data["files_info"].setdefault(fpath, {}) | ||
1672 | extended_data["files_info"][fpath]['size'] = fstat.st_size | ||
1673 | |||
1651 | if fstat.st_ino not in seen: | 1674 | if fstat.st_ino not in seen: |
1652 | seen.add(fstat.st_ino) | 1675 | seen.add(fstat.st_ino) |
1653 | total_size += fstat.st_size | 1676 | total_size += fstat.st_size |
1677 | |||
1678 | if fpath in pkgdebugsource: | ||
1679 | extended_data["files_info"][fpath]['debugsrc'] = pkgdebugsource[fpath] | ||
1680 | del pkgdebugsource[fpath] | ||
1681 | |||
1654 | d.setVar('FILES_INFO:' + pkg , json.dumps(files, sort_keys=True)) | 1682 | d.setVar('FILES_INFO:' + pkg , json.dumps(files, sort_keys=True)) |
1655 | 1683 | ||
1656 | process_postinst_on_target(pkg, d.getVar("MLPREFIX")) | 1684 | process_postinst_on_target(pkg, d.getVar("MLPREFIX")) |
@@ -1671,6 +1699,11 @@ fi | |||
1671 | 1699 | ||
1672 | sf.write('%s:%s: %d\n' % ('PKGSIZE', pkg, total_size)) | 1700 | sf.write('%s:%s: %d\n' % ('PKGSIZE', pkg, total_size)) |
1673 | 1701 | ||
1702 | subdata_extended_file = pkgdatadir + "/extended/%s.json.zstd" % pkg | ||
1703 | num_threads = int(d.getVar("BB_NUMBER_THREADS")) | ||
1704 | with bb.compress.zstd.open(subdata_extended_file, "wt", encoding="utf-8", num_threads=num_threads) as f: | ||
1705 | json.dump(extended_data, f, sort_keys=True, separators=(",", ":")) | ||
1706 | |||
1674 | # Symlinks needed for rprovides lookup | 1707 | # Symlinks needed for rprovides lookup |
1675 | rprov = d.getVar('RPROVIDES:%s' % pkg) or d.getVar('RPROVIDES') | 1708 | rprov = d.getVar('RPROVIDES:%s' % pkg) or d.getVar('RPROVIDES') |
1676 | if rprov: | 1709 | if rprov: |
@@ -1701,7 +1734,8 @@ fi | |||
1701 | write_extra_runtime_pkgs(global_variants, packages, pkgdatadir) | 1734 | write_extra_runtime_pkgs(global_variants, packages, pkgdatadir) |
1702 | 1735 | ||
1703 | } | 1736 | } |
1704 | emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides" | 1737 | emit_pkgdata[dirs] = "${PKGDESTWORK}/runtime ${PKGDESTWORK}/runtime-reverse ${PKGDESTWORK}/runtime-rprovides ${PKGDESTWORK}/extended" |
1738 | emit_pkgdata[vardepsexclude] = "BB_NUMBER_THREADS" | ||
1705 | 1739 | ||
1706 | ldconfig_postinst_fragment() { | 1740 | ldconfig_postinst_fragment() { |
1707 | if [ x"$D" = "x" ]; then | 1741 | if [ x"$D" = "x" ]; then |
diff --git a/meta/lib/oe/packagedata.py b/meta/lib/oe/packagedata.py index 0b17897e40..02c81e5a52 100644 --- a/meta/lib/oe/packagedata.py +++ b/meta/lib/oe/packagedata.py | |||
@@ -57,6 +57,18 @@ def read_subpkgdata_dict(pkg, d): | |||
57 | ret[newvar] = subd[var] | 57 | ret[newvar] = subd[var] |
58 | return ret | 58 | return ret |
59 | 59 | ||
60 | def read_subpkgdata_extended(pkg, d): | ||
61 | import json | ||
62 | import bb.compress.zstd | ||
63 | |||
64 | fn = d.expand("${PKGDATA_DIR}/extended/%s.json.zstd" % pkg) | ||
65 | try: | ||
66 | num_threads = int(d.getVar("BB_NUMBER_THREADS")) | ||
67 | with bb.compress.zstd.open(fn, "rt", encoding="utf-8", num_threads=num_threads) as f: | ||
68 | return json.load(f) | ||
69 | except FileNotFoundError: | ||
70 | return None | ||
71 | |||
60 | def _pkgmap(d): | 72 | def _pkgmap(d): |
61 | """Return a dictionary mapping package to recipe name.""" | 73 | """Return a dictionary mapping package to recipe name.""" |
62 | 74 | ||