From 2ee1fdb8404ddd70b2109d81431bd2814f098fc4 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Mon, 5 Dec 2022 23:00:19 +0100 Subject: bitbake: fetch2/git: Prevent git fetcher from fetching gitlab repository metadata The bitbake git fetcher currently fetches 'refs/*:refs/*', i.e. every single object in the remote repository. This works poorly with gitlab and github, which use the remote git repository to track its metadata like merge requests, CI pipelines and such. Specifically, gitlab generates refs/merge-requests/*, refs/pipelines/* and refs/keep-around/* and they all contain massive amount of data that are useless for the bitbake build purposes. The amount of useless data can in fact be so massive (e.g. with FDO mesa.git repository) that some proxies may outright terminate the 'git fetch' connection, and make it appear as if bitbake got stuck on 'git fetch' with no output. To avoid fetching all these useless metadata, tweak the git fetcher such that it only fetches refs/heads/* and refs/tags/* . Avoid using negative refspecs as those are only available in new git versions. Per feedback on the ML, Gerrit may push commits outsides of branches or tags during CI runs, which currently works with the 'nobranch=1' fetcher parameter. To retain this functionality, keep fetching everything in case the 'nobranch=1' is present. This still avoids fetching massive amount of data in the common case, since 'nobranch=1' is rare. Update 'nobranch' documentation. Reviewed-by: Peter Kjellerstedt (Bitbake rev: d32e5b0ec2ab85ffad7e56ac5b3160860b732556) Signed-off-by: Marek Vasut Signed-off-by: Alexandre Belloni Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/git.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'bitbake/lib/bb/fetch2/git.py') diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index 578edc5914..c80e8e5cf9 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py @@ -44,7 +44,7 @@ Supported SRC_URI options are: - nobranch Don't check the SHA validation for branch. set this option for the recipe - referring to commit which is valid in tag instead of branch. + referring to commit which is valid in any namespace instead of branch. The default is "0", set nobranch=1 if needed. - usehead @@ -382,7 +382,11 @@ class Git(FetchMethod): runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir) runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir) - fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) + + if ud.nobranch: + fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl)) + else: + fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl)) if ud.proto.lower() != 'file': bb.fetch2.check_network_access(d, fetch_cmd, ud.url) progresshandler = GitProgressHandler(d) -- cgit v1.2.3-54-g00ecf