diff options
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r-- | bitbake/lib/bb/fetch2/__init__.py | 78 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/crate.py | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/git.py | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/gitsm.py | 4 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/hg.py | 1 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/npm.py | 1 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/npmsw.py | 3 |
7 files changed, 91 insertions, 0 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index ffb1a92b26..35e9ca96b7 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py | |||
@@ -1579,6 +1579,7 @@ class FetchMethod(object): | |||
1579 | unpackdir = rootdir | 1579 | unpackdir = rootdir |
1580 | 1580 | ||
1581 | if not unpack or not cmd: | 1581 | if not unpack or not cmd: |
1582 | urldata.unpack_tracer.unpack("file-copy", unpackdir) | ||
1582 | # If file == dest, then avoid any copies, as we already put the file into dest! | 1583 | # If file == dest, then avoid any copies, as we already put the file into dest! |
1583 | dest = os.path.join(unpackdir, os.path.basename(file)) | 1584 | dest = os.path.join(unpackdir, os.path.basename(file)) |
1584 | if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)): | 1585 | if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)): |
@@ -1593,6 +1594,8 @@ class FetchMethod(object): | |||
1593 | destdir = urlpath.rsplit("/", 1)[0] + '/' | 1594 | destdir = urlpath.rsplit("/", 1)[0] + '/' |
1594 | bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir)) | 1595 | bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir)) |
1595 | cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir) | 1596 | cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir) |
1597 | else: | ||
1598 | urldata.unpack_tracer.unpack("archive-extract", unpackdir) | ||
1596 | 1599 | ||
1597 | if not cmd: | 1600 | if not cmd: |
1598 | return | 1601 | return |
@@ -1684,6 +1687,55 @@ class FetchMethod(object): | |||
1684 | """ | 1687 | """ |
1685 | return [] | 1688 | return [] |
1686 | 1689 | ||
1690 | |||
1691 | class DummyUnpackTracer(object): | ||
1692 | """ | ||
1693 | Abstract API definition for a class that traces unpacked source files back | ||
1694 | to their respective upstream SRC_URI entries, for software composition | ||
1695 | analysis, license compliance and detailed SBOM generation purposes. | ||
1696 | User may load their own unpack tracer class (instead of the dummy | ||
1697 | one) by setting the BB_UNPACK_TRACER_CLASS config parameter. | ||
1698 | """ | ||
1699 | def start(self, unpackdir, urldata_dict, d): | ||
1700 | """ | ||
1701 | Start tracing the core Fetch.unpack process, using an index to map | ||
1702 | unpacked files to each SRC_URI entry. | ||
1703 | This method is called by Fetch.unpack and it may receive nested calls by | ||
1704 | gitsm and npmsw fetchers, that expand SRC_URI entries by adding implicit | ||
1705 | URLs and by recursively calling Fetch.unpack from new (nested) Fetch | ||
1706 | instances. | ||
1707 | """ | ||
1708 | return | ||
1709 | def start_url(self, url): | ||
1710 | """Start tracing url unpack process. | ||
1711 | This method is called by Fetch.unpack before the fetcher-specific unpack | ||
1712 | method starts, and it may receive nested calls by gitsm and npmsw | ||
1713 | fetchers. | ||
1714 | """ | ||
1715 | return | ||
1716 | def unpack(self, unpack_type, destdir): | ||
1717 | """ | ||
1718 | Set unpack_type and destdir for current url. | ||
1719 | This method is called by the fetcher-specific unpack method after url | ||
1720 | tracing started. | ||
1721 | """ | ||
1722 | return | ||
1723 | def finish_url(self, url): | ||
1724 | """Finish tracing url unpack process and update the file index. | ||
1725 | This method is called by Fetch.unpack after the fetcher-specific unpack | ||
1726 | method finished its job, and it may receive nested calls by gitsm | ||
1727 | and npmsw fetchers. | ||
1728 | """ | ||
1729 | return | ||
1730 | def complete(self): | ||
1731 | """ | ||
1732 | Finish tracing the Fetch.unpack process, and check if all nested | ||
1733 | Fecth.unpack calls (if any) have been completed; if so, save collected | ||
1734 | metadata. | ||
1735 | """ | ||
1736 | return | ||
1737 | |||
1738 | |||
1687 | class Fetch(object): | 1739 | class Fetch(object): |
1688 | def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): | 1740 | def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): |
1689 | if localonly and cache: | 1741 | if localonly and cache: |
@@ -1704,10 +1756,30 @@ class Fetch(object): | |||
1704 | if key in urldata_cache: | 1756 | if key in urldata_cache: |
1705 | self.ud = urldata_cache[key] | 1757 | self.ud = urldata_cache[key] |
1706 | 1758 | ||
1759 | # the unpack_tracer object needs to be made available to possible nested | ||
1760 | # Fetch instances (when those are created by gitsm and npmsw fetchers) | ||
1761 | # so we set it as a global variable | ||
1762 | global unpack_tracer | ||
1763 | try: | ||
1764 | unpack_tracer | ||
1765 | except NameError: | ||
1766 | class_path = d.getVar("BB_UNPACK_TRACER_CLASS") | ||
1767 | if class_path: | ||
1768 | # use user-defined unpack tracer class | ||
1769 | import importlib | ||
1770 | module_name, _, class_name = class_path.rpartition(".") | ||
1771 | module = importlib.import_module(module_name) | ||
1772 | class_ = getattr(module, class_name) | ||
1773 | unpack_tracer = class_() | ||
1774 | else: | ||
1775 | # fall back to the dummy/abstract class | ||
1776 | unpack_tracer = DummyUnpackTracer() | ||
1777 | |||
1707 | for url in urls: | 1778 | for url in urls: |
1708 | if url not in self.ud: | 1779 | if url not in self.ud: |
1709 | try: | 1780 | try: |
1710 | self.ud[url] = FetchData(url, d, localonly) | 1781 | self.ud[url] = FetchData(url, d, localonly) |
1782 | self.ud[url].unpack_tracer = unpack_tracer | ||
1711 | except NonLocalMethod: | 1783 | except NonLocalMethod: |
1712 | if localonly: | 1784 | if localonly: |
1713 | self.ud[url] = None | 1785 | self.ud[url] = None |
@@ -1883,6 +1955,8 @@ class Fetch(object): | |||
1883 | if not urls: | 1955 | if not urls: |
1884 | urls = self.urls | 1956 | urls = self.urls |
1885 | 1957 | ||
1958 | unpack_tracer.start(root, self.ud, self.d) | ||
1959 | |||
1886 | for u in urls: | 1960 | for u in urls: |
1887 | ud = self.ud[u] | 1961 | ud = self.ud[u] |
1888 | ud.setup_localpath(self.d) | 1962 | ud.setup_localpath(self.d) |
@@ -1890,11 +1964,15 @@ class Fetch(object): | |||
1890 | if ud.lockfile: | 1964 | if ud.lockfile: |
1891 | lf = bb.utils.lockfile(ud.lockfile) | 1965 | lf = bb.utils.lockfile(ud.lockfile) |
1892 | 1966 | ||
1967 | unpack_tracer.start_url(u) | ||
1893 | ud.method.unpack(ud, root, self.d) | 1968 | ud.method.unpack(ud, root, self.d) |
1969 | unpack_tracer.finish_url(u) | ||
1894 | 1970 | ||
1895 | if ud.lockfile: | 1971 | if ud.lockfile: |
1896 | bb.utils.unlockfile(lf) | 1972 | bb.utils.unlockfile(lf) |
1897 | 1973 | ||
1974 | unpack_tracer.complete() | ||
1975 | |||
1898 | def clean(self, urls=None): | 1976 | def clean(self, urls=None): |
1899 | """ | 1977 | """ |
1900 | Clean files that the fetcher gets or places | 1978 | Clean files that the fetcher gets or places |
diff --git a/bitbake/lib/bb/fetch2/crate.py b/bitbake/lib/bb/fetch2/crate.py index 3310ed0050..01d49435c3 100644 --- a/bitbake/lib/bb/fetch2/crate.py +++ b/bitbake/lib/bb/fetch2/crate.py | |||
@@ -101,8 +101,10 @@ class Crate(Wget): | |||
101 | bp = d.getVar('BP') | 101 | bp = d.getVar('BP') |
102 | if bp == ud.parm.get('name'): | 102 | if bp == ud.parm.get('name'): |
103 | cmd = "tar -xz --no-same-owner -f %s" % thefile | 103 | cmd = "tar -xz --no-same-owner -f %s" % thefile |
104 | ud.unpack_tracer.unpack("crate-extract", rootdir) | ||
104 | else: | 105 | else: |
105 | cargo_bitbake = self._cargo_bitbake_path(rootdir) | 106 | cargo_bitbake = self._cargo_bitbake_path(rootdir) |
107 | ud.unpack_tracer.unpack("cargo-extract", cargo_bitbake) | ||
106 | 108 | ||
107 | cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_bitbake) | 109 | cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_bitbake) |
108 | 110 | ||
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index 4385d0b37a..c7ed1f0368 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py | |||
@@ -589,6 +589,8 @@ class Git(FetchMethod): | |||
589 | destdir = ud.destdir = os.path.join(destdir, destsuffix) | 589 | destdir = ud.destdir = os.path.join(destdir, destsuffix) |
590 | if os.path.exists(destdir): | 590 | if os.path.exists(destdir): |
591 | bb.utils.prunedir(destdir) | 591 | bb.utils.prunedir(destdir) |
592 | if not ud.bareclone: | ||
593 | ud.unpack_tracer.unpack("git", destdir) | ||
592 | 594 | ||
593 | need_lfs = self._need_lfs(ud) | 595 | need_lfs = self._need_lfs(ud) |
594 | 596 | ||
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py index a87361ccf3..f7f3af7212 100644 --- a/bitbake/lib/bb/fetch2/gitsm.py +++ b/bitbake/lib/bb/fetch2/gitsm.py | |||
@@ -218,6 +218,10 @@ class GitSM(Git): | |||
218 | 218 | ||
219 | try: | 219 | try: |
220 | newfetch = Fetch([url], d, cache=False) | 220 | newfetch = Fetch([url], d, cache=False) |
221 | # modpath is needed by unpack tracer to calculate submodule | ||
222 | # checkout dir | ||
223 | new_ud = newfetch.ud[url] | ||
224 | new_ud.modpath = modpath | ||
221 | newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) | 225 | newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module))) |
222 | except Exception as e: | 226 | except Exception as e: |
223 | logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) | 227 | logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e))) |
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py index 063e13008a..cbff8c490c 100644 --- a/bitbake/lib/bb/fetch2/hg.py +++ b/bitbake/lib/bb/fetch2/hg.py | |||
@@ -242,6 +242,7 @@ class Hg(FetchMethod): | |||
242 | revflag = "-r %s" % ud.revision | 242 | revflag = "-r %s" % ud.revision |
243 | subdir = ud.parm.get("destsuffix", ud.module) | 243 | subdir = ud.parm.get("destsuffix", ud.module) |
244 | codir = "%s/%s" % (destdir, subdir) | 244 | codir = "%s/%s" % (destdir, subdir) |
245 | ud.unpack_tracer.unpack("hg", codir) | ||
245 | 246 | ||
246 | scmdata = ud.parm.get("scmdata", "") | 247 | scmdata = ud.parm.get("scmdata", "") |
247 | if scmdata != "nokeep": | 248 | if scmdata != "nokeep": |
diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py index f83485ad85..15f3f19bc8 100644 --- a/bitbake/lib/bb/fetch2/npm.py +++ b/bitbake/lib/bb/fetch2/npm.py | |||
@@ -298,6 +298,7 @@ class Npm(FetchMethod): | |||
298 | destsuffix = ud.parm.get("destsuffix", "npm") | 298 | destsuffix = ud.parm.get("destsuffix", "npm") |
299 | destdir = os.path.join(rootdir, destsuffix) | 299 | destdir = os.path.join(rootdir, destsuffix) |
300 | npm_unpack(ud.localpath, destdir, d) | 300 | npm_unpack(ud.localpath, destdir, d) |
301 | ud.unpack_tracer.unpack("npm", destdir) | ||
301 | 302 | ||
302 | def clean(self, ud, d): | 303 | def clean(self, ud, d): |
303 | """Clean any existing full or partial download""" | 304 | """Clean any existing full or partial download""" |
diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py index 4ff2c8ffc3..ff5f8dc755 100644 --- a/bitbake/lib/bb/fetch2/npmsw.py +++ b/bitbake/lib/bb/fetch2/npmsw.py | |||
@@ -191,7 +191,9 @@ class NpmShrinkWrap(FetchMethod): | |||
191 | else: | 191 | else: |
192 | raise ParameterError("Unsupported dependency: %s" % name, ud.url) | 192 | raise ParameterError("Unsupported dependency: %s" % name, ud.url) |
193 | 193 | ||
194 | # name is needed by unpack tracer for module mapping | ||
194 | ud.deps.append({ | 195 | ud.deps.append({ |
196 | "name": name, | ||
195 | "url": url, | 197 | "url": url, |
196 | "localpath": localpath, | 198 | "localpath": localpath, |
197 | "extrapaths": extrapaths, | 199 | "extrapaths": extrapaths, |
@@ -270,6 +272,7 @@ class NpmShrinkWrap(FetchMethod): | |||
270 | destsuffix = ud.parm.get("destsuffix") | 272 | destsuffix = ud.parm.get("destsuffix") |
271 | if destsuffix: | 273 | if destsuffix: |
272 | destdir = os.path.join(rootdir, destsuffix) | 274 | destdir = os.path.join(rootdir, destsuffix) |
275 | ud.unpack_tracer.unpack("npm-shrinkwrap", destdir) | ||
273 | 276 | ||
274 | bb.utils.mkdirhier(destdir) | 277 | bb.utils.mkdirhier(destdir) |
275 | bb.utils.copyfile(ud.shrinkwrap_file, | 278 | bb.utils.copyfile(ud.shrinkwrap_file, |