diff options
author | Christian Lindeberg <christian.lindeberg@axis.com> | 2024-09-06 11:27:39 +0200 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2024-09-10 13:05:00 +0100 |
commit | 87c29b5a947956ab64c3fe881dfbc7eee5fef9e4 (patch) | |
tree | a9e82dc5b5fb14230bb62d07d32eccc348cf9399 | |
parent | dd7631426c1a7d0e6777540c81646777e712be26 (diff) | |
download | poky-87c29b5a947956ab64c3fe881dfbc7eee5fef9e4.tar.gz |
bitbake: fetch2: Add gomodgit fetcher
Add a go module fetcher for downloading module dependencies to the
module cache directly from a git repository. The fetcher can be used
with the go-mod class in OE-Core.
A module dependency can be specified with:
SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;srcrev=..."
(Bitbake rev: 29ff38ccf0d5389a5bee81e252a78548361a9d7c)
Signed-off-by: Christian Lindeberg <christian.lindeberg@axis.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | bitbake/lib/bb/fetch2/__init__.py | 1 | ||||
-rw-r--r-- | bitbake/lib/bb/fetch2/gomod.py | 140 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/fetch.py | 89 |
3 files changed, 228 insertions, 2 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index f84ce59992..ddee4400bb 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py | |||
@@ -2112,3 +2112,4 @@ methods.append(az.Az()) | |||
2112 | methods.append(crate.Crate()) | 2112 | methods.append(crate.Crate()) |
2113 | methods.append(gcp.GCP()) | 2113 | methods.append(gcp.GCP()) |
2114 | methods.append(gomod.GoMod()) | 2114 | methods.append(gomod.GoMod()) |
2115 | methods.append(gomod.GoModGit()) | ||
diff --git a/bitbake/lib/bb/fetch2/gomod.py b/bitbake/lib/bb/fetch2/gomod.py index fe025e367f..1b532d03ff 100644 --- a/bitbake/lib/bb/fetch2/gomod.py +++ b/bitbake/lib/bb/fetch2/gomod.py | |||
@@ -1,12 +1,13 @@ | |||
1 | """ | 1 | """ |
2 | BitBake 'Fetch' implementation for Go modules | 2 | BitBake 'Fetch' implementation for Go modules |
3 | 3 | ||
4 | The gomod fetcher is used to download Go modules to the module cache from a | 4 | The gomod/gomodgit fetchers are used to download Go modules to the module cache |
5 | module proxy. | 5 | from a module proxy or directly from a version control repository. |
6 | 6 | ||
7 | Example SRC_URI: | 7 | Example SRC_URI: |
8 | 8 | ||
9 | SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..." | 9 | SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..." |
10 | SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;repo=go.googlesource.com/net;srcrev=..." | ||
10 | 11 | ||
11 | Required SRC_URI parameters: | 12 | Required SRC_URI parameters: |
12 | 13 | ||
@@ -27,6 +28,23 @@ Optional SRC_URI parameters: | |||
27 | only the go.mod file. Alternatively, set the SRC_URI varible flag for | 28 | only the go.mod file. Alternatively, set the SRC_URI varible flag for |
28 | "module@version.sha256sum". | 29 | "module@version.sha256sum". |
29 | 30 | ||
31 | - protocol | ||
32 | The method used when fetching directly from a version control repository. | ||
33 | The default is "https" for git. | ||
34 | |||
35 | - repo | ||
36 | The URL when fetching directly from a version control repository. Required | ||
37 | when the URL is different from the module path. | ||
38 | |||
39 | - srcrev | ||
40 | The revision identifier used when fetching directly from a version control | ||
41 | repository. Alternatively, set the SRCREV varible for "module@version". | ||
42 | |||
43 | - subdir | ||
44 | The module subdirectory when fetching directly from a version control | ||
45 | repository. Required when the module is not located in the root of the | ||
46 | repository. | ||
47 | |||
30 | Related variables: | 48 | Related variables: |
31 | 49 | ||
32 | - GO_MOD_PROXY | 50 | - GO_MOD_PROXY |
@@ -41,14 +59,19 @@ See the Go modules reference, https://go.dev/ref/mod, for more information | |||
41 | about the module cache, module proxies and version control systems. | 59 | about the module cache, module proxies and version control systems. |
42 | """ | 60 | """ |
43 | 61 | ||
62 | import hashlib | ||
44 | import os | 63 | import os |
45 | import re | 64 | import re |
46 | import shutil | 65 | import shutil |
66 | import subprocess | ||
47 | import zipfile | 67 | import zipfile |
48 | 68 | ||
49 | import bb | 69 | import bb |
50 | from bb.fetch2 import FetchError | 70 | from bb.fetch2 import FetchError |
51 | from bb.fetch2 import MissingParameterError | 71 | from bb.fetch2 import MissingParameterError |
72 | from bb.fetch2 import runfetchcmd | ||
73 | from bb.fetch2 import subprocess_setup | ||
74 | from bb.fetch2.git import Git | ||
52 | from bb.fetch2.wget import Wget | 75 | from bb.fetch2.wget import Wget |
53 | 76 | ||
54 | 77 | ||
@@ -126,3 +149,116 @@ class GoMod(Wget): | |||
126 | # If the module does not have a go.mod file, synthesize | 149 | # If the module does not have a go.mod file, synthesize |
127 | # one containing only a module statement. | 150 | # one containing only a module statement. |
128 | mf.write(f'module {module}\n'.encode()) | 151 | mf.write(f'module {module}\n'.encode()) |
152 | |||
153 | |||
154 | class GoModGit(Git): | ||
155 | """Class to fetch Go modules directly from a git repository""" | ||
156 | |||
157 | def supports(self, ud, d): | ||
158 | """Check to see if a given URL is for this fetcher.""" | ||
159 | return ud.type == 'gomodgit' | ||
160 | |||
161 | def urldata_init(self, ud, d): | ||
162 | """Set up to download the module from the git repository. | ||
163 | |||
164 | Set up to download the git repository to the module cache directory and | ||
165 | unpack the module zip file and the go.mod file: | ||
166 | |||
167 | cache/vcs/<hash>: The bare git repository. | ||
168 | cache/download/<module>/@v/<version>.zip: The module zip file. | ||
169 | cache/download/<module>/@v/<version>.mod: The go.mod file. | ||
170 | """ | ||
171 | |||
172 | moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod' | ||
173 | |||
174 | if 'version' not in ud.parm: | ||
175 | raise MissingParameterError('version', ud.url) | ||
176 | |||
177 | module = ud.host + ud.path | ||
178 | ud.parm['module'] = module | ||
179 | |||
180 | # Set host, path and srcrev for git download | ||
181 | if 'repo' in ud.parm: | ||
182 | repo = ud.parm['repo'] | ||
183 | idx = repo.find('/') | ||
184 | if idx != -1: | ||
185 | ud.host = repo[:idx] | ||
186 | ud.path = repo[idx:] | ||
187 | else: | ||
188 | ud.host = repo | ||
189 | ud.path = '' | ||
190 | if 'protocol' not in ud.parm: | ||
191 | ud.parm['protocol'] = 'https' | ||
192 | name = f"{module}@{ud.parm['version']}" | ||
193 | ud.names = [name] | ||
194 | srcrev = d.getVar('SRCREV_' + name) | ||
195 | if srcrev: | ||
196 | if 'srcrev' not in ud.parm: | ||
197 | ud.parm['srcrev'] = srcrev | ||
198 | else: | ||
199 | if 'srcrev' in ud.parm: | ||
200 | d.setVar('SRCREV_' + name, ud.parm['srcrev']) | ||
201 | if 'branch' not in ud.parm: | ||
202 | ud.parm['nobranch'] = '1' | ||
203 | |||
204 | # Set subpath, subdir and bareclone for git unpack | ||
205 | if 'subdir' in ud.parm: | ||
206 | ud.parm['subpath'] = ud.parm['subdir'] | ||
207 | key = f"git3:{ud.parm['protocol']}://{ud.host}{ud.path}".encode() | ||
208 | ud.parm['key'] = key | ||
209 | ud.parm['subdir'] = os.path.join(moddir, 'cache/vcs', | ||
210 | hashlib.sha256(key).hexdigest()) | ||
211 | ud.parm['bareclone'] = '1' | ||
212 | |||
213 | super().urldata_init(ud, d) | ||
214 | |||
215 | def unpack(self, ud, rootdir, d): | ||
216 | """Unpack the module in the module cache.""" | ||
217 | |||
218 | # Unpack the bare git repository | ||
219 | super().unpack(ud, rootdir, d) | ||
220 | |||
221 | moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod' | ||
222 | |||
223 | # Create the info file | ||
224 | module = ud.parm['module'] | ||
225 | repodir = os.path.join(rootdir, ud.parm['subdir']) | ||
226 | with open(repodir + '.info', 'wb') as f: | ||
227 | f.write(ud.parm['key']) | ||
228 | |||
229 | # Unpack the go.mod file from the repository | ||
230 | unpackdir = os.path.join(rootdir, moddir, 'cache/download', | ||
231 | escape(module), '@v') | ||
232 | bb.utils.mkdirhier(unpackdir) | ||
233 | srcrev = ud.parm['srcrev'] | ||
234 | version = ud.parm['version'] | ||
235 | escaped_version = escape(version) | ||
236 | cmd = f"git ls-tree -r --name-only '{srcrev}'" | ||
237 | if 'subpath' in ud.parm: | ||
238 | cmd += f" '{ud.parm['subpath']}'" | ||
239 | files = runfetchcmd(cmd, d, workdir=repodir).split() | ||
240 | name = escaped_version + '.mod' | ||
241 | bb.note(f"Unpacking {name} to {unpackdir}/") | ||
242 | with open(os.path.join(unpackdir, name), mode='wb') as mf: | ||
243 | f = 'go.mod' | ||
244 | if 'subpath' in ud.parm: | ||
245 | f = os.path.join(ud.parm['subpath'], f) | ||
246 | if f in files: | ||
247 | cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f] | ||
248 | subprocess.check_call(cmd, stdout=mf, cwd=repodir, | ||
249 | preexec_fn=subprocess_setup) | ||
250 | else: | ||
251 | # If the module does not have a go.mod file, synthesize one | ||
252 | # containing only a module statement. | ||
253 | mf.write(f'module {module}\n'.encode()) | ||
254 | |||
255 | # Synthesize the module zip file from the repository | ||
256 | name = escaped_version + '.zip' | ||
257 | bb.note(f"Unpacking {name} to {unpackdir}/") | ||
258 | with zipfile.ZipFile(os.path.join(unpackdir, name), mode='w') as zf: | ||
259 | prefix = module + '@' + version + '/' | ||
260 | for f in files: | ||
261 | cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f] | ||
262 | data = subprocess.check_output(cmd, cwd=repodir, | ||
263 | preexec_fn=subprocess_setup) | ||
264 | zf.writestr(prefix + f, data) | ||
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py index 2365a50960..832e0dd6a4 100644 --- a/bitbake/lib/bb/tests/fetch.py +++ b/bitbake/lib/bb/tests/fetch.py | |||
@@ -3455,3 +3455,92 @@ class GoModTest(FetcherTest): | |||
3455 | downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') | 3455 | downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') |
3456 | self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) | 3456 | self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) |
3457 | self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) | 3457 | self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) |
3458 | |||
3459 | class GoModGitTest(FetcherTest): | ||
3460 | |||
3461 | @skipIfNoNetwork() | ||
3462 | def test_gomodgit_url_repo(self): | ||
3463 | urls = ['gomodgit://golang.org/x/net;version=v0.9.0;' | ||
3464 | 'repo=go.googlesource.com/net;' | ||
3465 | 'srcrev=694cff8668bac64e0864b552bffc280cd27f21b1'] | ||
3466 | |||
3467 | fetcher = bb.fetch2.Fetch(urls, self.d) | ||
3468 | ud = fetcher.ud[urls[0]] | ||
3469 | self.assertEqual(ud.host, 'go.googlesource.com') | ||
3470 | self.assertEqual(ud.path, '/net') | ||
3471 | self.assertEqual(ud.names, ['golang.org/x/net@v0.9.0']) | ||
3472 | self.assertEqual(self.d.getVar('SRCREV_golang.org/x/net@v0.9.0'), '694cff8668bac64e0864b552bffc280cd27f21b1') | ||
3473 | |||
3474 | fetcher.download() | ||
3475 | self.assertTrue(os.path.exists(ud.localpath)) | ||
3476 | |||
3477 | fetcher.unpack(self.unpackdir) | ||
3478 | vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') | ||
3479 | self.assertTrue(os.path.exists(os.path.join(vcsdir, 'ed42bd05533fd84ae290a5d33ebd3695a0a2b06131beebd5450825bee8603aca'))) | ||
3480 | downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') | ||
3481 | self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.zip'))) | ||
3482 | self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod'))) | ||
3483 | |||
3484 | @skipIfNoNetwork() | ||
3485 | def test_gomodgit_url_subdir(self): | ||
3486 | urls = ['gomodgit://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;' | ||
3487 | 'repo=github.com/Azure/azure-sdk-for-go;subdir=sdk/storage/azblob;' | ||
3488 | 'srcrev=ec928e0ed34db682b3f783d3739d1c538142e0c3'] | ||
3489 | |||
3490 | fetcher = bb.fetch2.Fetch(urls, self.d) | ||
3491 | ud = fetcher.ud[urls[0]] | ||
3492 | self.assertEqual(ud.host, 'github.com') | ||
3493 | self.assertEqual(ud.path, '/Azure/azure-sdk-for-go') | ||
3494 | self.assertEqual(ud.parm['subpath'], 'sdk/storage/azblob') | ||
3495 | self.assertEqual(ud.names, ['github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0']) | ||
3496 | self.assertEqual(self.d.getVar('SRCREV_github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'), 'ec928e0ed34db682b3f783d3739d1c538142e0c3') | ||
3497 | |||
3498 | fetcher.download() | ||
3499 | self.assertTrue(os.path.exists(ud.localpath)) | ||
3500 | |||
3501 | fetcher.unpack(self.unpackdir) | ||
3502 | vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') | ||
3503 | self.assertTrue(os.path.exists(os.path.join(vcsdir, 'd31d6145676ed3066ce573a8198f326dea5be45a43b3d8f41ce7787fd71d66b3'))) | ||
3504 | downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') | ||
3505 | self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip'))) | ||
3506 | self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod'))) | ||
3507 | |||
3508 | @skipIfNoNetwork() | ||
3509 | def test_gomodgit_url_srcrev_var(self): | ||
3510 | urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0'] | ||
3511 | self.d.setVar('SRCREV_gopkg.in/ini.v1@v1.67.0', 'b2f570e5b5b844226bbefe6fb521d891f529a951') | ||
3512 | |||
3513 | fetcher = bb.fetch2.Fetch(urls, self.d) | ||
3514 | ud = fetcher.ud[urls[0]] | ||
3515 | self.assertEqual(ud.host, 'gopkg.in') | ||
3516 | self.assertEqual(ud.path, '/ini.v1') | ||
3517 | self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0']) | ||
3518 | self.assertEqual(ud.parm['srcrev'], 'b2f570e5b5b844226bbefe6fb521d891f529a951') | ||
3519 | |||
3520 | fetcher.download() | ||
3521 | fetcher.unpack(self.unpackdir) | ||
3522 | vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') | ||
3523 | self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3'))) | ||
3524 | downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') | ||
3525 | self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) | ||
3526 | self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) | ||
3527 | |||
3528 | @skipIfNoNetwork() | ||
3529 | def test_gomodgit_url_no_go_mod_in_module(self): | ||
3530 | urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0;' | ||
3531 | 'srcrev=b2f570e5b5b844226bbefe6fb521d891f529a951'] | ||
3532 | |||
3533 | fetcher = bb.fetch2.Fetch(urls, self.d) | ||
3534 | ud = fetcher.ud[urls[0]] | ||
3535 | self.assertEqual(ud.host, 'gopkg.in') | ||
3536 | self.assertEqual(ud.path, '/ini.v1') | ||
3537 | self.assertEqual(ud.names, ['gopkg.in/ini.v1@v1.67.0']) | ||
3538 | self.assertEqual(self.d.getVar('SRCREV_gopkg.in/ini.v1@v1.67.0'), 'b2f570e5b5b844226bbefe6fb521d891f529a951') | ||
3539 | |||
3540 | fetcher.download() | ||
3541 | fetcher.unpack(self.unpackdir) | ||
3542 | vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs') | ||
3543 | self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3'))) | ||
3544 | downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download') | ||
3545 | self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip'))) | ||
3546 | self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod'))) | ||