16 files changed, 888 insertions, 482 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 5bf2c4b8cf..0ad987c596 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -23,17 +23,18 @@ import collections
 import subprocess
 import pickle
 import errno
-import bb.persist_data, bb.utils
+import bb.utils
 import bb.checksum
 import bb.process
 import bb.event
 __version__ = "2"
 _checksum_cache = bb.checksum.FileChecksumCache()
+_revisions_cache = bb.checksum.RevisionsCache()
 logger = logging.getLogger("BitBake.Fetcher")
-CHECKSUM_LIST = [ "md5", "sha256", "sha1", "sha384", "sha512" ]
+CHECKSUM_LIST = [ "goh1", "md5", "sha256", "sha1", "sha384", "sha512" ]
 SHOWN_CHECKSUM_LIST = ["sha256"]
 class BBFetchException(Exception):
@@ -237,7 +238,7 @@ class URI(object):
        # to RFC compliant URL format. E.g.:
        #   file://foo.diff -> file:foo.diff
        if urlp.scheme in self._netloc_forbidden:
-            uri = re.sub("(?<=:)//(?!/)", "", uri, 1)
+            uri = re.sub(r"(?<=:)//(?!/)", "", uri, count=1)
            reparse = 1
        if reparse:
@@ -352,6 +353,14 @@ def decodeurl(url):
    user, password, parameters).
    """
+    uri = URI(url)
+    path = uri.path if uri.path else "/"
+    return uri.scheme, uri.hostport, path, uri.username, uri.password, uri.params
+def decodemirrorurl(url):
+    """Decodes a mirror URL into the tokens (scheme, network location, path,
+    user, password, parameters).
+    """
    m = re.compile('(?P<type>[^:]*)://((?P<user>[^/;]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url)
    if not m:
        raise MalformedUrl(url)
@@ -370,6 +379,9 @@ def decodeurl(url):
    elif type.lower() == 'file':
        host = ""
        path = location
+        if user:
+            path = user + '@' + path
+            user = ""
    else:
        host = location
        path = "/"
@@ -402,32 +414,34 @@ def encodeurl(decoded):
    if not type:
        raise MissingParameterError('type', "encoded from the data %s" % str(decoded))
-    url = ['%s://' % type]
+    uri = URI()
+    uri.scheme = type
    if user and type != "file":
-        url.append("%s" % user)
+        uri.username = user
        if pswd:
-            url.append(":%s" % pswd)
+            uri.password = pswd
-        url.append("@")
    if host and type != "file":
-        url.append("%s" % host)
+        uri.hostname = host
    if path:
        # Standardise path to ensure comparisons work
        while '//' in path:
            path = path.replace("//", "/")
-        url.append("%s" % urllib.parse.quote(path))
+        uri.path = path
+        if type == "file":
+            # Use old not IETF compliant style
+            uri.relative = False
    if p:
-        for parm in p:
+        uri.params = p
-            url.append(";%s=%s" % (parm, p[parm]))
-    return "".join(url)
+    return str(uri)
 def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
    if not ud.url or not uri_find or not uri_replace:
        logger.error("uri_replace: passed an undefined value, not replacing")
        return None
-    uri_decoded = list(decodeurl(ud.url))
+    uri_decoded = list(decodemirrorurl(ud.url))
-    uri_find_decoded = list(decodeurl(uri_find))
+    uri_find_decoded = list(decodemirrorurl(uri_find))
-    uri_replace_decoded = list(decodeurl(uri_replace))
+    uri_replace_decoded = list(decodemirrorurl(uri_replace))
    logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded))
    result_decoded = ['', '', '', '', '', {}]
    # 0 - type, 1 - host, 2 - path, 3 - user,  4- pswd, 5 - params
@@ -460,7 +474,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
                for k in replacements:
                    uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k])
                #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc]))
-                result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1)
+                result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], count=1)
            if loc == 2:
                # Handle path manipulations
                basename = None
@@ -493,18 +507,23 @@ methods = []
 urldata_cache = {}
 saved_headrevs = {}
-def fetcher_init(d):
+def fetcher_init(d, servercontext=True):
    """
    Called to initialize the fetchers once the configuration data is known.
    Calls before this must not hit the cache.
    """
-    revs = bb.persist_data.persist('BB_URI_HEADREVS', d)
+    _checksum_cache.init_cache(d.getVar("BB_CACHEDIR"))
+    _revisions_cache.init_cache(d.getVar("BB_CACHEDIR"))
+    if not servercontext:
+        return
    try:
        # fetcher_init is called multiple times, so make sure we only save the
        # revs the first time it is called.
        if not bb.fetch2.saved_headrevs:
-            bb.fetch2.saved_headrevs = dict(revs)
+            bb.fetch2.saved_headrevs = _revisions_cache.get_revs()
    except:
        pass
@@ -514,11 +533,10 @@ def fetcher_init(d):
        logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
    elif srcrev_policy == "clear":
        logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy)
-        revs.clear()
+        _revisions_cache.clear_cache()
    else:
        raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
-    _checksum_cache.init_cache(d.getVar("BB_CACHEDIR"))
    for m in methods:
        if hasattr(m, "init"):
@@ -526,9 +544,11 @@ def fetcher_init(d):
 def fetcher_parse_save():
    _checksum_cache.save_extras()
+    _revisions_cache.save_extras()
 def fetcher_parse_done():
    _checksum_cache.save_merge()
+    _revisions_cache.save_merge()
 def fetcher_compare_revisions(d):
    """
@@ -536,7 +556,7 @@ def fetcher_compare_revisions(d):
    when bitbake was started and return true if they have changed.
    """
-    headrevs = dict(bb.persist_data.persist('BB_URI_HEADREVS', d))
+    headrevs = _revisions_cache.get_revs()
    return headrevs != bb.fetch2.saved_headrevs
 def mirror_from_string(data):
@@ -786,8 +806,8 @@ def _get_srcrev(d, method_name='sortable_revision'):
        return "", revs
-    if len(scms) == 1 and len(urldata[scms[0]].names) == 1:
+    if len(scms) == 1:
-        autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0])
+        autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].name)
        revs.append(rev)
        if len(rev) > 10:
            rev = rev[:10]
@@ -808,13 +828,12 @@ def _get_srcrev(d, method_name='sortable_revision'):
    seenautoinc = False
    for scm in scms:
        ud = urldata[scm]
-        for name in ud.names:
+        autoinc, rev = getattr(ud.method, method_name)(ud, d, ud.name)
-            autoinc, rev = getattr(ud.method, method_name)(ud, d, name)
+        revs.append(rev)
-            revs.append(rev)
+        seenautoinc = seenautoinc or autoinc
-            seenautoinc = seenautoinc or autoinc
+        if len(rev) > 10:
-            if len(rev) > 10:
+            rev = rev[:10]
-                rev = rev[:10]
+        name_to_rev[ud.name] = rev
-            name_to_rev[name] = rev
    # Replace names by revisions in the SRCREV_FORMAT string. The approach used
    # here can handle names being prefixes of other names and names appearing
    # as substrings in revisions (in which case the name should not be
@@ -878,6 +897,7 @@ FETCH_EXPORT_VARS = ['HOME', 'PATH',
                     'AWS_SESSION_TOKEN',
                     'GIT_CACHE_PATH',
                     'REMOTE_CONTAINERS_IPC',
+                     'GITHUB_TOKEN',
                     'SSL_CERT_DIR']
 def get_fetcher_environment(d):
@@ -1072,6 +1092,10 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
        # If that tarball is a local file:// we need to provide a symlink to it
        dldir = ld.getVar("DL_DIR")
+        if bb.utils.to_boolean(ld.getVar("BB_FETCH_PREMIRRORONLY")):
+            ld = ld.createCopy()
+            ld.setVar("BB_NO_NETWORK", "1")
        if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
            # Create donestamp in old format to avoid triggering a re-download
            if ud.donestamp:
@@ -1093,7 +1117,10 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
                    origud.method.build_mirror_data(origud, ld)
            return origud.localpath
        # Otherwise the result is a local file:// and we symlink to it
-        ensure_symlink(ud.localpath, origud.localpath)
+        # This may also be a link to a shallow archive
+        # When using shallow mode, add a symlink to the original fullshallow
+        # path to ensure a valid symlink even in the `PREMIRRORS` case
+        origud.method.update_mirror_links(ud, origud)
        update_stamp(origud, ld)
        return ud.localpath
@@ -1127,25 +1154,6 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
        if ud.lockfile and ud.lockfile != origud.lockfile:
            bb.utils.unlockfile(lf)
-def ensure_symlink(target, link_name):
-    if not os.path.exists(link_name):
-        dirname = os.path.dirname(link_name)
-        bb.utils.mkdirhier(dirname)
-        if os.path.islink(link_name):
-            # Broken symbolic link
-            os.unlink(link_name)
-        # In case this is executing without any file locks held (as is
-        # the case for file:// URLs), two tasks may end up here at the
-        # same time, in which case we do not want the second task to
-        # fail when the link has already been created by the first task.
-        try:
-            os.symlink(target, link_name)
-        except FileExistsError:
-            pass
 def try_mirrors(fetch, d, origud, mirrors, check = False):
    """
    Try to use a mirrored version of the sources.
@@ -1174,7 +1182,7 @@ def trusted_network(d, url):
    if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")):
        return True
-    pkgname = d.expand(d.getVar('PN', False))
+    pkgname = d.getVar('PN')
    trusted_hosts = None
    if pkgname:
        trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False)
@@ -1227,20 +1235,17 @@ def srcrev_internal_helper(ud, d, name):
        if srcrev and srcrev != "INVALID":
            break
-    if 'rev' in ud.parm and 'tag' in ud.parm:
+    if 'rev' in ud.parm:
-        raise FetchError("Please specify a ;rev= parameter or a ;tag= parameter in the url %s but not both." % (ud.url))
+        parmrev = ud.parm['rev']
-    if 'rev' in ud.parm or 'tag' in ud.parm:
-        if 'rev' in ud.parm:
-            parmrev = ud.parm['rev']
-        else:
-            parmrev = ud.parm['tag']
        if srcrev == "INVALID" or not srcrev:
            return parmrev
        if srcrev != parmrev:
            raise FetchError("Conflicting revisions (%s from SRCREV and %s from the url) found, please specify one valid value" % (srcrev, parmrev))
        return parmrev
+    if 'tag' in ud.parm and (srcrev == "INVALID" or not srcrev):
+        return ud.parm['tag']
    if srcrev == "INVALID" or not srcrev:
        raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url)
    if srcrev == "AUTOINC":
@@ -1263,7 +1268,7 @@ def get_checksum_file_list(d):
            found = False
            paths = ud.method.localfile_searchpaths(ud, d)
            for f in paths:
-                pth = ud.decodedurl
+                pth = ud.path
                if os.path.exists(f):
                    found = True
                filelist.append(f + ":" + str(os.path.exists(f)))
@@ -1308,23 +1313,28 @@ class FetchData(object):
        self.setup = False
        def configure_checksum(checksum_id):
+            checksum_plain_name = "%ssum" % checksum_id
            if "name" in self.parm:
                checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id)
            else:
-                checksum_name = "%ssum" % checksum_id
+                checksum_name = checksum_plain_name
-            setattr(self, "%s_name" % checksum_id, checksum_name)
            if checksum_name in self.parm:
                checksum_expected = self.parm[checksum_name]
-            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs"]:
+            elif checksum_plain_name in self.parm:
+                checksum_expected = self.parm[checksum_plain_name]
+                checksum_name = checksum_plain_name
+            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs", "gomod", "npm"]:
                checksum_expected = None
            else:
                checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
+            setattr(self, "%s_name" % checksum_id, checksum_name)
            setattr(self, "%s_expected" % checksum_id, checksum_expected)
-        self.names = self.parm.get("name",'default').split(',')
+        self.name = self.parm.get("name",'default')
+        if "," in self.name:
+            raise ParameterError("The fetcher no longer supports multiple name parameters in a single url", self.url)
        self.method = None
        for m in methods:
@@ -1376,13 +1386,7 @@ class FetchData(object):
        self.lockfile = basepath + '.lock'
    def setup_revisions(self, d):
-        self.revisions = {}
+        self.revision = srcrev_internal_helper(self, d, self.name)
-        for name in self.names:
-            self.revisions[name] = srcrev_internal_helper(self, d, name)
-        # add compatibility code for non name specified case
-        if len(self.names) == 1:
-            self.revision = self.revisions[self.names[0]]
    def setup_localpath(self, d):
        if not self.localpath:
@@ -1510,7 +1514,7 @@ class FetchMethod(object):
                     (file, urldata.parm.get('unpack')))
        base, ext = os.path.splitext(file)
-        if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz']:
+        if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz', '.zst']:
            efile = os.path.join(rootdir, os.path.basename(base))
        else:
            efile = file
@@ -1569,11 +1573,11 @@ class FetchMethod(object):
                datafile = None
                if output:
                    for line in output.decode().splitlines():
-                        if line.startswith('data.tar.'):
+                        if line.startswith('data.tar.') or line == 'data.tar':
                            datafile = line
                            break
                    else:
-                        raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar.* file", urldata.url)
+                        raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar* file", urldata.url)
                else:
                    raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url)
                cmd = 'ar x %s %s && %s -p -f %s && rm %s' % (file, datafile, tar_cmd, datafile, datafile)
@@ -1606,7 +1610,7 @@ class FetchMethod(object):
                    if urlpath.find("/") != -1:
                        destdir = urlpath.rsplit("/", 1)[0] + '/'
                        bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir))
-                cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir)
+                cmd = 'cp --force --preserve=timestamps --no-dereference --recursive -H "%s" "%s"' % (file, destdir)
        else:
            urldata.unpack_tracer.unpack("archive-extract", unpackdir)
@@ -1635,6 +1639,28 @@ class FetchMethod(object):
        """
        bb.utils.remove(urldata.localpath)
+    def ensure_symlink(self, target, link_name):
+        if not os.path.exists(link_name):
+            dirname = os.path.dirname(link_name)
+            bb.utils.mkdirhier(dirname)
+            if os.path.islink(link_name):
+                # Broken symbolic link
+                os.unlink(link_name)
+            # In case this is executing without any file locks held (as is
+            # the case for file:// URLs), two tasks may end up here at the
+            # same time, in which case we do not want the second task to
+            # fail when the link has already been created by the first task.
+            try:
+                os.symlink(target, link_name)
+            except FileExistsError:
+                pass
+    def update_mirror_links(self, ud, origud):
+        # For local file:// results, create a symlink to them
+        # This may also be a link to a shallow archive
+        self.ensure_symlink(ud.localpath, origud.localpath)
    def try_premirror(self, urldata, d):
        """
        Should premirrors be used?
@@ -1662,13 +1688,13 @@ class FetchMethod(object):
        if not hasattr(self, "_latest_revision"):
            raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url)
-        revs = bb.persist_data.persist('BB_URI_HEADREVS', d)
        key = self.generate_revision_key(ud, d, name)
-        try:
-            return revs[key]
+        rev = _revisions_cache.get_rev(key)
-        except KeyError:
+        if rev is None:
-            revs[key] = rev = self._latest_revision(ud, d, name)
+            rev = self._latest_revision(ud, d, name)
-            return rev
+            _revisions_cache.set_rev(key, rev)
+        return rev
    def sortable_revision(self, ud, d, name):
        latest_rev = self._build_revision(ud, d, name)
@@ -1806,7 +1832,7 @@ class Fetch(object):
            self.ud[url] = FetchData(url, self.d)
        self.ud[url].setup_localpath(self.d)
-        return self.d.expand(self.ud[url].localpath)
+        return self.ud[url].localpath
    def localpaths(self):
        """
@@ -1859,25 +1885,28 @@ class Fetch(object):
                            logger.debug(str(e))
                            done = False
+                d = self.d
                if premirroronly:
-                    self.d.setVar("BB_NO_NETWORK", "1")
+                    # Only disable the network in a copy
+                    d = bb.data.createCopy(self.d)
+                    d.setVar("BB_NO_NETWORK", "1")
                firsterr = None
                verified_stamp = False
                if done:
-                    verified_stamp = m.verify_donestamp(ud, self.d)
+                    verified_stamp = m.verify_donestamp(ud, d)
-                if not done and (not verified_stamp or m.need_update(ud, self.d)):
+                if not done and (not verified_stamp or m.need_update(ud, d)):
                    try:
-                        if not trusted_network(self.d, ud.url):
+                        if not trusted_network(d, ud.url):
                            raise UntrustedUrl(ud.url)
                        logger.debug("Trying Upstream")
-                        m.download(ud, self.d)
+                        m.download(ud, d)
                        if hasattr(m, "build_mirror_data"):
-                            m.build_mirror_data(ud, self.d)
+                            m.build_mirror_data(ud, d)
                        done = True
                        # early checksum verify, so that if checksum mismatched,
                        # fetcher still have chance to fetch from mirror
-                        m.update_donestamp(ud, self.d)
+                        m.update_donestamp(ud, d)
                    except bb.fetch2.NetworkAccess:
                        raise
@@ -1896,17 +1925,17 @@ class Fetch(object):
                        firsterr = e
                        # Remove any incomplete fetch
                        if not verified_stamp and m.cleanup_upon_failure():
-                            m.clean(ud, self.d)
+                            m.clean(ud, d)
                        logger.debug("Trying MIRRORS")
-                        mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
+                        mirrors = mirror_from_string(d.getVar('MIRRORS'))
-                        done = m.try_mirrors(self, ud, self.d, mirrors)
+                        done = m.try_mirrors(self, ud, d, mirrors)
-                if not done or not m.done(ud, self.d):
+                if not done or not m.done(ud, d):
                    if firsterr:
                        logger.error(str(firsterr))
                    raise FetchError("Unable to fetch URL from any source.", u)
-                m.update_donestamp(ud, self.d)
+                m.update_donestamp(ud, d)
            except IOError as e:
                if e.errno in [errno.ESTALE]:
@@ -2088,6 +2117,7 @@ from . import npmsw
 from . import az
 from . import crate
 from . import gcp
+from . import gomod
 methods.append(local.Local())
 methods.append(wget.Wget())
@@ -2110,3 +2140,5 @@ methods.append(npmsw.NpmShrinkWrap())
 methods.append(az.Az())
 methods.append(crate.Crate())
 methods.append(gcp.GCP())
+methods.append(gomod.GoMod())
+methods.append(gomod.GoModGit())
diff --git a/bitbake/lib/bb/fetch2/az.py b/bitbake/lib/bb/fetch2/az.py
index 3ccc594c22..1d3664f213 100644
--- a/bitbake/lib/bb/fetch2/az.py
+++ b/bitbake/lib/bb/fetch2/az.py
@@ -36,6 +36,8 @@ class Az(Wget):
        az_sas = d.getVar('AZ_SAS')
        if az_sas and az_sas not in ud.url:
+            if not az_sas.startswith('?'):
+                raise FetchError("When using AZ_SAS, it must start with a '?' character to mark the start of the query-parameters.")
            ud.url += az_sas
        return Wget.checkstatus(self, fetch, ud, d, try_again)
@@ -62,15 +64,18 @@ class Az(Wget):
        az_sas = d.getVar('AZ_SAS')
        if az_sas:
+            if not az_sas.startswith('?'):
+                raise FetchError("When using AZ_SAS, it must start with a '?' character to mark the start of the query-parameters.")
            azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas)
        else:
            azuri = '%s%s%s' % ('https://', ud.host, ud.path)
+        dldir = d.getVar("DL_DIR")
        if os.path.exists(ud.localpath):
            # file exists, but we didnt complete it.. trying again.
-            fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % azuri)
+            fetchcmd += " -c -P %s '%s'" % (dldir, azuri)
        else:
-            fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % azuri)
+            fetchcmd += " -P %s '%s'" % (dldir, azuri)
        try:
            self._runwget(ud, d, fetchcmd, False)
diff --git a/bitbake/lib/bb/fetch2/clearcase.py b/bitbake/lib/bb/fetch2/clearcase.py
index 1a9c863769..17500daf95 100644
--- a/bitbake/lib/bb/fetch2/clearcase.py
+++ b/bitbake/lib/bb/fetch2/clearcase.py
@@ -108,7 +108,7 @@ class ClearCase(FetchMethod):
                                                ud.module.replace("/", "."),
                                                ud.label.replace("/", "."))
-        ud.viewname         = "%s-view%s" % (ud.identifier, d.getVar("DATETIME", d, True))
+        ud.viewname         = "%s-view%s" % (ud.identifier, d.getVar("DATETIME"))
        ud.csname           = "%s-config-spec" % (ud.identifier)
        ud.ccasedir         = os.path.join(d.getVar("DL_DIR"), ud.type)
        ud.viewdir          = os.path.join(ud.ccasedir, ud.viewname)
@@ -130,8 +130,6 @@ class ClearCase(FetchMethod):
        self.debug("configspecfile  = %s" % ud.configspecfile)
        self.debug("localfile       = %s" % ud.localfile)
-        ud.localfile = os.path.join(d.getVar("DL_DIR"), ud.localfile)
    def _build_ccase_command(self, ud, command):
        """
        Build up a commandline based on ud
@@ -196,7 +194,7 @@ class ClearCase(FetchMethod):
    def need_update(self, ud, d):
        if ("LATEST" in ud.label) or (ud.customspec and "LATEST" in ud.customspec):
-            ud.identifier += "-%s" % d.getVar("DATETIME",d, True)
+            ud.identifier += "-%s" % d.getVar("DATETIME")
            return True
        if os.path.exists(ud.localpath):
            return False
diff --git a/bitbake/lib/bb/fetch2/crate.py b/bitbake/lib/bb/fetch2/crate.py
index 01d49435c3..e611736f06 100644
--- a/bitbake/lib/bb/fetch2/crate.py
+++ b/bitbake/lib/bb/fetch2/crate.py
@@ -70,6 +70,7 @@ class Crate(Wget):
            host = 'crates.io/api/v1/crates'
        ud.url = "https://%s/%s/%s/download" % (host, name, version)
+        ud.versionsurl = "https://%s/%s/versions" % (host, name)
        ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
        if 'name' not in ud.parm:
            ud.parm['name'] = '%s-%s' % (name, version)
@@ -139,3 +140,11 @@ class Crate(Wget):
            mdpath = os.path.join(bbpath, cratepath, mdfile)
            with open(mdpath, "w") as f:
                json.dump(metadata, f)
+    def latest_versionstring(self, ud, d):
+        from functools import cmp_to_key
+        json_data = json.loads(self._fetch_index(ud.versionsurl, ud, d))
+        versions = [(0, i["num"], "") for i in json_data["versions"]]
+        versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp))
+        return (versions[-1][1], "")
diff --git a/bitbake/lib/bb/fetch2/gcp.py b/bitbake/lib/bb/fetch2/gcp.py
index f40ce2eaa5..86546d40bf 100644
--- a/bitbake/lib/bb/fetch2/gcp.py
+++ b/bitbake/lib/bb/fetch2/gcp.py
@@ -46,8 +46,7 @@ class GCP(FetchMethod):
        else:
            ud.basename = os.path.basename(ud.path)
-        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+        ud.localfile = ud.basename
-        ud.basecmd = "gsutil stat"
    def get_gcp_client(self):
        from google.cloud import storage
@@ -58,17 +57,20 @@ class GCP(FetchMethod):
        Fetch urls using the GCP API.
        Assumes localpath was called first.
        """
+        from google.api_core.exceptions import NotFound
        logger.debug2(f"Trying to download gs://{ud.host}{ud.path} to {ud.localpath}")
        if self.gcp_client is None:
            self.get_gcp_client()
-        bb.fetch2.check_network_access(d, ud.basecmd, f"gs://{ud.host}{ud.path}")
+        bb.fetch2.check_network_access(d, "blob.download_to_filename", f"gs://{ud.host}{ud.path}")
-        runfetchcmd("%s %s" % (ud.basecmd, f"gs://{ud.host}{ud.path}"), d)
        # Path sometimes has leading slash, so strip it
        path = ud.path.lstrip("/")
        blob = self.gcp_client.bucket(ud.host).blob(path)
-        blob.download_to_filename(ud.localpath)
+        try:
+            blob.download_to_filename(ud.localpath)
+        except NotFound:
+            raise FetchError("The GCP API threw a NotFound exception")
        # Additional sanity checks copied from the wget class (although there
        # are no known issues which mean these are required, treat the GCP API
@@ -90,8 +92,7 @@ class GCP(FetchMethod):
        if self.gcp_client is None:
            self.get_gcp_client()
-        bb.fetch2.check_network_access(d, ud.basecmd, f"gs://{ud.host}{ud.path}")
+        bb.fetch2.check_network_access(d, "gcp_client.bucket(ud.host).blob(path).exists()", f"gs://{ud.host}{ud.path}")
-        runfetchcmd("%s %s" % (ud.basecmd, f"gs://{ud.host}{ud.path}"), d)
        # Path sometimes has leading slash, so strip it
        path = ud.path.lstrip("/")
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index c7ff769fdf..14ec45a3f6 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -9,15 +9,6 @@ Supported SRC_URI options are:
 - branch
   The git branch to retrieve from. The default is "master"
-   This option also supports multiple branch fetching, with branches
-   separated by commas.  In multiple branches case, the name option
-   must have the same number of names to match the branches, which is
-   used to specify the SRC_REV for the branch
-   e.g:
-   SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
-   SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
-   SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY"
 - tag
    The git tag to retrieve. The default is "master"
@@ -81,6 +72,7 @@ import shlex
 import shutil
 import subprocess
 import tempfile
+import urllib
 import bb
 import bb.progress
 from contextlib import contextmanager
@@ -190,14 +182,11 @@ class Git(FetchMethod):
        ud.bareclone = ud.parm.get("bareclone","0") == "1"
        if ud.bareclone:
            ud.nocheckout = 1
-  
-        ud.unresolvedrev = {}
+        ud.unresolvedrev = ""
-        branches = ud.parm.get("branch", "").split(',')
+        ud.branch = ud.parm.get("branch", "")
-        if branches == [""] and not ud.nobranch:
+        if not ud.branch and not ud.nobranch:
-            bb.warn("URL: %s does not set any branch parameter. The future default branch used by tools and repositories is uncertain and we will therefore soon require this is set in all git urls." % ud.url)
+            raise bb.fetch2.ParameterError("The url does not set any branch parameter or set nobranch=1.", ud.url)
-            branches = ["master"]
-        if len(branches) != len(ud.names):
-            raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
        ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
@@ -207,8 +196,11 @@ class Git(FetchMethod):
        if ud.bareclone:
            ud.cloneflags += " --mirror"
+        ud.shallow_skip_fast = False
        ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
        ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split()
+        if 'tag' in ud.parm:
+            ud.shallow_extra_refs.append("refs/tags/" + ud.parm['tag'])
        depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH")
        if depth_default is not None:
@@ -225,32 +217,27 @@ class Git(FetchMethod):
        revs_default = d.getVar("BB_GIT_SHALLOW_REVS")
        ud.shallow_revs = []
-        ud.branches = {}
-        for pos, name in enumerate(ud.names):
+        ud.unresolvedrev = ud.branch
-            branch = branches[pos]
-            ud.branches[name] = branch
+        shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % ud.name)
-            ud.unresolvedrev[name] = branch
+        if shallow_depth is not None:
+            try:
-            shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name)
+                shallow_depth = int(shallow_depth or 0)
-            if shallow_depth is not None:
+            except ValueError:
-                try:
+                raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
-                    shallow_depth = int(shallow_depth or 0)
+            else:
-                except ValueError:
+                if shallow_depth < 0:
-                    raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
+                    raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
-                else:
+                ud.shallow_depths[ud.name] = shallow_depth
-                    if shallow_depth < 0:
-                        raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
+        revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % ud.name)
-                    ud.shallow_depths[name] = shallow_depth
+        if revs is not None:
+            ud.shallow_revs.extend(revs.split())
-            revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name)
+        elif revs_default is not None:
-            if revs is not None:
+            ud.shallow_revs.extend(revs_default.split())
-                ud.shallow_revs.extend(revs.split())
-            elif revs_default is not None:
+        if ud.shallow and not ud.shallow_revs and ud.shallow_depths[ud.name] == 0:
-                ud.shallow_revs.extend(revs_default.split())
-        if (ud.shallow and
-                not ud.shallow_revs and
-                all(ud.shallow_depths[n] == 0 for n in ud.names)):
            # Shallow disabled for this URL
            ud.shallow = False
@@ -259,10 +246,9 @@ class Git(FetchMethod):
            # rev of this repository. This will get resolved into a revision
            # later. If an actual revision happens to have also been provided
            # then this setting will be overridden.
-            for name in ud.names:
+            ud.unresolvedrev = 'HEAD'
-                ud.unresolvedrev[name] = 'HEAD'
-        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all"
+        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all -c clone.defaultRemoteName=origin"
        write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
        ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
@@ -270,12 +256,11 @@ class Git(FetchMethod):
        ud.setup_revisions(d)
-        for name in ud.names:
+        # Ensure any revision that doesn't look like a SHA-1 is translated into one
-            # Ensure any revision that doesn't look like a SHA-1 is translated into one
+        if not sha1_re.match(ud.revision or ''):
-            if not sha1_re.match(ud.revisions[name] or ''):
+            if ud.revision:
-                if ud.revisions[name]:
+                ud.unresolvedrev = ud.revision
-                    ud.unresolvedrev[name] = ud.revisions[name]
+            ud.revision = self.latest_revision(ud, d, ud.name)
-                ud.revisions[name] = self.latest_revision(ud, d, name)
        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
        if gitsrcname.startswith('.'):
@@ -286,8 +271,7 @@ class Git(FetchMethod):
        # upstream repo in the future, the mirror will remain intact and still
        # contain the revision
        if ud.rebaseable:
-            for name in ud.names:
+            gitsrcname = gitsrcname + '_' + ud.revision
-                gitsrcname = gitsrcname + '_' + ud.revisions[name]
        dl_dir = d.getVar("DL_DIR")
        gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
@@ -305,15 +289,14 @@ class Git(FetchMethod):
            if ud.shallow_revs:
                tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
-            for name, revision in sorted(ud.revisions.items()):
+            tarballname = "%s_%s" % (tarballname, ud.revision[:7])
-                tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7])
+            depth = ud.shallow_depths[ud.name]
-                depth = ud.shallow_depths[name]
+            if depth:
-                if depth:
+                tarballname = "%s-%s" % (tarballname, depth)
-                    tarballname = "%s-%s" % (tarballname, depth)
            shallow_refs = []
            if not ud.nobranch:
-                shallow_refs.extend(ud.branches.values())
+                shallow_refs.append(ud.branch)
            if ud.shallow_extra_refs:
                shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs)
            if shallow_refs:
@@ -338,18 +321,19 @@ class Git(FetchMethod):
            return True
        if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
            return True
-        for name in ud.names:
+        if not self._contains_ref(ud, d, ud.name, ud.clonedir):
-            if not self._contains_ref(ud, d, name, ud.clonedir):
+            return True
-                return True
        return False
    def lfs_need_update(self, ud, d):
+        if not self._need_lfs(ud):
+            return False
        if self.clonedir_need_update(ud, d):
            return True
-        for name in ud.names:
+        if not self._lfs_objects_downloaded(ud, d, ud.clonedir):
-            if not self._lfs_objects_downloaded(ud, d, name, ud.clonedir):
+            return True
-                return True
        return False
    def clonedir_need_shallow_revs(self, ud, d):
@@ -366,6 +350,13 @@ class Git(FetchMethod):
    def tarball_need_update(self, ud):
        return ud.write_tarballs and not os.path.exists(ud.fullmirror)
+    def update_mirror_links(self, ud, origud):
+        super().update_mirror_links(ud, origud)
+        # When using shallow mode, add a symlink to the original fullshallow
+        # path to ensure a valid symlink even in the `PREMIRRORS` case
+        if ud.shallow and not os.path.exists(origud.fullshallow):
+            self.ensure_symlink(ud.localpath, origud.fullshallow)
    def try_premirror(self, ud, d):
        # If we don't do this, updating an existing checkout with only premirrors
        # is not possible
@@ -446,6 +437,24 @@ class Git(FetchMethod):
            if ud.proto.lower() != 'file':
                bb.fetch2.check_network_access(d, clone_cmd, ud.url)
            progresshandler = GitProgressHandler(d)
+            # Try creating a fast initial shallow clone
+            # Enabling ud.shallow_skip_fast will skip this
+            # If the Git error "Server does not allow request for unadvertised object"
+            # occurs, shallow_skip_fast is enabled automatically.
+            # This may happen if the Git server does not allow the request
+            # or if the Git client has issues with this functionality.
+            if ud.shallow and not ud.shallow_skip_fast:
+                try:
+                    self.clone_shallow_with_tarball(ud, d)
+                    # When the shallow clone has succeeded, use the shallow tarball
+                    ud.localpath = ud.fullshallow
+                    return
+                except:
+                    logger.warning("Creating fast initial shallow clone failed, try initial regular clone now.")
+            # When skipping fast initial shallow or the fast inital shallow clone failed:
+            # Try again with an initial regular clone
            runfetchcmd(clone_cmd, d, log=progresshandler)
        # Update the checkout if needed
@@ -473,9 +482,8 @@ class Git(FetchMethod):
                if exc.errno != errno.ENOENT:
                    raise
-        for name in ud.names:
+        if not self._contains_ref(ud, d, ud.name, ud.clonedir):
-            if not self._contains_ref(ud, d, name, ud.clonedir):
+            raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revision, ud.branch))
-                raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
        if ud.shallow and ud.write_shallow_tarballs:
            missing_rev = self.clonedir_need_shallow_revs(ud, d)
@@ -483,128 +491,168 @@ class Git(FetchMethod):
                raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
        if self.lfs_need_update(ud, d):
-            # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
+            self.lfs_fetch(ud, d, ud.clonedir, ud.revision)
-            # of all LFS blobs needed at the srcrev.
-            #
-            # It would be nice to just do this inline here by running 'git-lfs fetch'
-            # on the bare clonedir, but that operation requires a working copy on some
-            # releases of Git LFS.
-            with tempfile.TemporaryDirectory(dir=d.getVar('DL_DIR')) as tmpdir:
-                # Do the checkout. This implicitly involves a Git LFS fetch.
-                Git.unpack(self, ud, tmpdir, d)
-                # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
-                # the bare clonedir.
-                #
-                # As this procedure is invoked repeatedly on incremental fetches as
-                # a recipe's SRCREV is bumped throughout its lifetime, this will
-                # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs
-                # corresponding to all the blobs reachable from the different revs
-                # fetched across time.
-                #
-                # Only do this if the unpack resulted in a .git/lfs directory being
-                # created; this only happens if at least one blob needed to be
-                # downloaded.
-                if os.path.exists(os.path.join(ud.destdir, ".git", "lfs")):
-                    runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/.git" % ud.destdir)
-    def build_mirror_data(self, ud, d):
-        # Create as a temp file and move atomically into position to avoid races
+    def lfs_fetch(self, ud, d, clonedir, revision, fetchall=False, progresshandler=None):
-        @contextmanager
+        """Helper method for fetching Git LFS data"""
-        def create_atomic(filename):
+        try:
-            fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename))
+            if self._need_lfs(ud) and self._contains_lfs(ud, d, clonedir) and len(revision):
-            try:
+                self._ensure_git_lfs(d, ud)
-                yield tfile
-                umask = os.umask(0o666)
+                # Using worktree with the revision because .lfsconfig may exists
-                os.umask(umask)
+                worktree_add_cmd = "%s worktree add wt %s" % (ud.basecmd, revision)
-                os.chmod(tfile, (0o666 & ~umask))
+                runfetchcmd(worktree_add_cmd, d, log=progresshandler, workdir=clonedir)
-                os.rename(tfile, filename)
+                lfs_fetch_cmd = "%s lfs fetch %s" % (ud.basecmd, "--all" if fetchall else "")
-            finally:
+                runfetchcmd(lfs_fetch_cmd, d, log=progresshandler, workdir=(clonedir + "/wt"))
-                os.close(fd)
+                worktree_rem_cmd = "%s worktree remove -f wt" % ud.basecmd
+                runfetchcmd(worktree_rem_cmd, d, log=progresshandler, workdir=clonedir)
+        except:
+            logger.warning("Fetching LFS did not succeed.")
+    @contextmanager
+    def create_atomic(self, filename):
+        """Create as a temp file and move atomically into position to avoid races"""
+        fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename))
+        try:
+            yield tfile
+            umask = os.umask(0o666)
+            os.umask(umask)
+            os.chmod(tfile, (0o666 & ~umask))
+            os.rename(tfile, filename)
+        finally:
+            os.close(fd)
+    def build_mirror_data(self, ud, d):
        if ud.shallow and ud.write_shallow_tarballs:
            if not os.path.exists(ud.fullshallow):
                if os.path.islink(ud.fullshallow):
                    os.unlink(ud.fullshallow)
-                tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
+                self.clone_shallow_with_tarball(ud, d)
-                shallowclone = os.path.join(tempdir, 'git')
-                try:
-                    self.clone_shallow_local(ud, shallowclone, d)
-                    logger.info("Creating tarball of git repository")
-                    with create_atomic(ud.fullshallow) as tfile:
-                        runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone)
-                    runfetchcmd("touch %s.done" % ud.fullshallow, d)
-                finally:
-                    bb.utils.remove(tempdir, recurse=True)
        elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
            if os.path.islink(ud.fullmirror):
                os.unlink(ud.fullmirror)
            logger.info("Creating tarball of git repository")
-            with create_atomic(ud.fullmirror) as tfile:
+            with self.create_atomic(ud.fullmirror) as tfile:
                mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d,
                        quiet=True, workdir=ud.clonedir)
                runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ."
                        % (tfile, mtime), d, workdir=ud.clonedir)
            runfetchcmd("touch %s.done" % ud.fullmirror, d)
+    def clone_shallow_with_tarball(self, ud, d):
+        ret = False
+        tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
+        shallowclone = os.path.join(tempdir, 'git')
+        try:
+            try:
+                self.clone_shallow_local(ud, shallowclone, d)
+            except:
+                logger.warning("Fast shallow clone failed, try to skip fast mode now.")
+                bb.utils.remove(tempdir, recurse=True)
+                os.mkdir(tempdir)
+                ud.shallow_skip_fast = True
+                self.clone_shallow_local(ud, shallowclone, d)
+            logger.info("Creating tarball of git repository")
+            with self.create_atomic(ud.fullshallow) as tfile:
+                runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone)
+            runfetchcmd("touch %s.done" % ud.fullshallow, d)
+            ret = True
+        finally:
+            bb.utils.remove(tempdir, recurse=True)
+        return ret
    def clone_shallow_local(self, ud, dest, d):
-        """Clone the repo and make it shallow.
+        """
+        Shallow fetch from ud.clonedir (${DL_DIR}/git2/<gitrepo> by default):
+        - For BB_GIT_SHALLOW_DEPTH: git fetch --depth <depth> rev
+        - For BB_GIT_SHALLOW_REVS: git fetch --shallow-exclude=<revs> rev
+        """
-        The upstream url of the new clone isn't set at this time, as it'll be
+        progresshandler = GitProgressHandler(d)
-        set correctly when unpacked."""
+        repourl = self._get_repo_url(ud)
-        runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d)
+        bb.utils.mkdirhier(dest)
+        init_cmd = "%s init -q" % ud.basecmd
+        if ud.bareclone:
+            init_cmd += " --bare"
+        runfetchcmd(init_cmd, d, workdir=dest)
+        # Use repourl when creating a fast initial shallow clone
+        # Prefer already existing full bare clones if available
+        if not ud.shallow_skip_fast and not os.path.exists(ud.clonedir):
+            remote = shlex.quote(repourl)
+        else:
+            remote = ud.clonedir
+        runfetchcmd("%s remote add origin %s" % (ud.basecmd, remote), d, workdir=dest)
-        to_parse, shallow_branches = [], []
+        # Check the histories which should be excluded
-        for name in ud.names:
+        shallow_exclude = ''
-            revision = ud.revisions[name]
+        for revision in ud.shallow_revs:
-            depth = ud.shallow_depths[name]
+            shallow_exclude += " --shallow-exclude=%s" % revision
-            if depth:
-                to_parse.append('%s~%d^{}' % (revision, depth - 1))
-            # For nobranch, we need a ref, otherwise the commits will be
+        revision = ud.revision
-            # removed, and for non-nobranch, we truncate the branch to our
+        depth = ud.shallow_depths[ud.name]
-            # srcrev, to avoid keeping unnecessary history beyond that.
-            branch = ud.branches[name]
-            if ud.nobranch:
-                ref = "refs/shallow/%s" % name
-            elif ud.bareclone:
-                ref = "refs/heads/%s" % branch
-            else:
-                ref = "refs/remotes/origin/%s" % branch
-            shallow_branches.append(ref)
+        # The --depth and --shallow-exclude can't be used together
-            runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
+        if depth and shallow_exclude:
+            raise bb.fetch2.FetchError("BB_GIT_SHALLOW_REVS is set, but BB_GIT_SHALLOW_DEPTH is not 0.")
+        # For nobranch, we need a ref, otherwise the commits will be
+        # removed, and for non-nobranch, we truncate the branch to our
+        # srcrev, to avoid keeping unnecessary history beyond that.
+        branch = ud.branch
+        if ud.nobranch:
+            ref = "refs/shallow/%s" % ud.name
+        elif ud.bareclone:
+            ref = "refs/heads/%s" % branch
+        else:
+            ref = "refs/remotes/origin/%s" % branch
+        fetch_cmd = "%s fetch origin %s" % (ud.basecmd, revision)
+        if depth:
+            fetch_cmd += " --depth %s" % depth
+        if shallow_exclude:
+            fetch_cmd += shallow_exclude
-        # Map srcrev+depths to revisions
+        # Advertise the revision for lower version git such as 2.25.1:
-        parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest)
+        # error: Server does not allow request for unadvertised object.
+        # The ud.clonedir is a local temporary dir, will be removed when
+        # fetch is done, so we can do anything on it.
+        adv_cmd = 'git branch -f advertise-%s %s' % (revision, revision)
+        if ud.shallow_skip_fast:
+            runfetchcmd(adv_cmd, d, workdir=ud.clonedir)
-        # Resolve specified revisions
+        runfetchcmd(fetch_cmd, d, workdir=dest)
-        parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest)
+        runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
-        shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines()
+        # Fetch Git LFS data
+        self.lfs_fetch(ud, d, dest, ud.revision)
        # Apply extra ref wildcards
-        all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd,
+        all_refs_remote = runfetchcmd("%s ls-remote origin 'refs/*'" % ud.basecmd, \
-                               d, workdir=dest).splitlines()
+                                        d, workdir=dest).splitlines()
+        all_refs = []
+        for line in all_refs_remote:
+            all_refs.append(line.split()[-1])
+        extra_refs = []
        for r in ud.shallow_extra_refs:
            if not ud.bareclone:
                r = r.replace('refs/heads/', 'refs/remotes/origin/')
            if '*' in r:
                matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs)
-                shallow_branches.extend(matches)
+                extra_refs.extend(matches)
            else:
-                shallow_branches.append(r)
+                extra_refs.append(r)
-        # Make the repository shallow
+        for ref in extra_refs:
-        shallow_cmd = [self.make_shallow_path, '-s']
+            ref_fetch = ref.replace('refs/heads/', '').replace('refs/remotes/origin/', '').replace('refs/tags/', '')
-        for b in shallow_branches:
+            runfetchcmd("%s fetch origin --depth 1 %s" % (ud.basecmd, ref_fetch), d, workdir=dest)
-            shallow_cmd.append('-r')
+            revision = runfetchcmd("%s rev-parse FETCH_HEAD" % ud.basecmd, d, workdir=dest)
-            shallow_cmd.append(b)
+            runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
-        shallow_cmd.extend(shallow_revisions)
-        runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest)
+        # The url is local ud.clonedir, set it to upstream one
+        runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=dest)
    def unpack(self, ud, destdir, d):
        """ unpack the downloaded src to destdir"""
@@ -612,7 +660,7 @@ class Git(FetchMethod):
        subdir = ud.parm.get("subdir")
        subpath = ud.parm.get("subpath")
        readpathspec = ""
-        def_destsuffix = "git/"
+        def_destsuffix = (d.getVar("BB_GIT_DEFAULT_DESTSUFFIX") or "git") + "/"
        if subpath:
            readpathspec = ":%s" % subpath
@@ -664,30 +712,43 @@ class Git(FetchMethod):
        if not source_found:
            raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)
+        # If there is a tag parameter in the url and we also have a fixed srcrev, check the tag
+        # matches the revision
+        if 'tag' in ud.parm and sha1_re.match(ud.revision):
+            output = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.parm['tag']), d, workdir=destdir)
+            output = output.strip()
+            if output != ud.revision:
+                # It is possible ud.revision is the revision on an annotated tag which won't match the output of rev-list
+                # If it resolves to the same thing there isn't a problem.
+                output2 = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.revision), d, workdir=destdir)
+                output2 = output2.strip()
+                if output != output2:
+                    raise bb.fetch2.FetchError("The revision the git tag '%s' resolved to didn't match the SRCREV in use (%s vs %s)" % (ud.parm['tag'], output, ud.revision), ud.url)
        repourl = self._get_repo_url(ud)
        runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir)
        if self._contains_lfs(ud, d, destdir):
-            if need_lfs and not self._find_git_lfs(d):
+            if not need_lfs:
-                raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
-            elif not need_lfs:
                bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
            else:
+                self._ensure_git_lfs(d, ud)
                runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir)
        if not ud.nocheckout:
            if subpath:
-                runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d,
+                runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revision, readpathspec), d,
                            workdir=destdir)
                runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
            elif not ud.nobranch:
-                branchname =  ud.branches[ud.names[0]]
+                branchname =  ud.branch
                runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
-                            ud.revisions[ud.names[0]]), d, workdir=destdir)
+                            ud.revision), d, workdir=destdir)
                runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
                            branchname), d, workdir=destdir)
            else:
-                runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir)
+                runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revision), d, workdir=destdir)
        return True
@@ -701,8 +762,13 @@ class Git(FetchMethod):
            clonedir = os.path.realpath(ud.localpath)
            to_remove.append(clonedir)
+        # Remove shallow mirror tarball
+        if ud.shallow:
+            to_remove.append(ud.fullshallow)
+            to_remove.append(ud.fullshallow + ".done")
        for r in to_remove:
-            if os.path.exists(r):
+            if os.path.exists(r) or os.path.islink(r):
                bb.note('Removing %s' % r)
                bb.utils.remove(r, True)
@@ -713,10 +779,10 @@ class Git(FetchMethod):
        cmd = ""
        if ud.nobranch:
            cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
-                ud.basecmd, ud.revisions[name])
+                ud.basecmd, ud.revision)
        else:
            cmd =  "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
-                ud.basecmd, ud.revisions[name], ud.branches[name])
+                ud.basecmd, ud.revision, ud.branch)
        try:
            output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
        except bb.fetch2.FetchError:
@@ -725,19 +791,21 @@ class Git(FetchMethod):
            raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
        return output.split()[0] != "0"
-    def _lfs_objects_downloaded(self, ud, d, name, wd):
+    def _lfs_objects_downloaded(self, ud, d, wd):
        """
        Verifies whether the LFS objects for requested revisions have already been downloaded
        """
        # Bail out early if this repository doesn't use LFS
-        if not self._need_lfs(ud) or not self._contains_lfs(ud, d, wd):
+        if not self._contains_lfs(ud, d, wd):
            return True
+        self._ensure_git_lfs(d, ud)
        # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file
        # existence.
        # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git
        cmd = "%s lfs ls-files -l %s" \
-                % (ud.basecmd, ud.revisions[name])
+                % (ud.basecmd, ud.revision)
        output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip()
        # Do not do any further matching if no objects are managed by LFS
        if not output:
@@ -761,18 +829,8 @@ class Git(FetchMethod):
        """
        Check if the repository has 'lfs' (large file) content
        """
-        if ud.nobranch:
-            # If no branch is specified, use the current git commit
-            refname = self._build_revision(ud, d, ud.names[0])
-        elif wd == ud.clonedir:
-            # The bare clonedir doesn't use the remote names; it has the branch immediately.
-            refname = ud.branches[ud.names[0]]
-        else:
-            refname = "origin/%s" % ud.branches[ud.names[0]]
        cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
-            ud.basecmd, refname)
+            ud.basecmd, ud.revision)
        try:
            output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
@@ -782,12 +840,14 @@ class Git(FetchMethod):
            pass
        return False
-    def _find_git_lfs(self, d):
+    def _ensure_git_lfs(self, d, ud):
        """
-        Return True if git-lfs can be found, False otherwise.
+        Ensures that git-lfs is available, raising a FetchError if it isn't.
        """
-        import shutil
+        if shutil.which("git-lfs", path=d.getVar('PATH')) is None:
-        return shutil.which("git-lfs", path=d.getVar('PATH')) is not None
+            raise bb.fetch2.FetchError(
+                "Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 "
+                "to ignore it)" % self._get_repo_url(ud))
    def _get_repo_url(self, ud):
        """
@@ -795,21 +855,21 @@ class Git(FetchMethod):
        """
        # Note that we do not support passwords directly in the git urls. There are several
        # reasons. SRC_URI can be written out to things like buildhistory and people don't
-        # want to leak passwords like that. Its also all too easy to share metadata without 
+        # want to leak passwords like that. Its also all too easy to share metadata without
        # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as
        # alternatives so we will not take patches adding password support here.
        if ud.user:
            username = ud.user + '@'
        else:
            username = ""
-        return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path)
+        return "%s://%s%s%s" % (ud.proto, username, ud.host, urllib.parse.quote(ud.path))
    def _revision_key(self, ud, d, name):
        """
        Return a unique key for the url
        """
        # Collapse adjacent slashes
-        return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
+        return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev
    def _lsremote(self, ud, d, search):
        """
@@ -842,26 +902,26 @@ class Git(FetchMethod):
        Compute the HEAD revision for the url
        """
        if not d.getVar("__BBSRCREV_SEEN"):
-            raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev[name], ud.host+ud.path))
+            raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev, ud.host+ud.path))
        # Ensure we mark as not cached
        bb.fetch2.mark_recipe_nocache(d)
        output = self._lsremote(ud, d, "")
        # Tags of the form ^{} may not work, need to fallback to other form
-        if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:
+        if ud.unresolvedrev[:5] == "refs/" or ud.usehead:
-            head = ud.unresolvedrev[name]
+            head = ud.unresolvedrev
-            tag = ud.unresolvedrev[name]
+            tag = ud.unresolvedrev
        else:
-            head = "refs/heads/%s" % ud.unresolvedrev[name]
+            head = "refs/heads/%s" % ud.unresolvedrev
-            tag = "refs/tags/%s" % ud.unresolvedrev[name]
+            tag = "refs/tags/%s" % ud.unresolvedrev
        for s in [head, tag + "^{}", tag]:
            for l in output.strip().split('\n'):
                sha1, ref = l.split()
                if s == ref:
                    return sha1
        raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
-            (ud.unresolvedrev[name], ud.host+ud.path))
+            (ud.unresolvedrev, ud.host+ud.path))
    def latest_versionstring(self, ud, d):
        """
@@ -912,23 +972,22 @@ class Git(FetchMethod):
        return pupver
    def _build_revision(self, ud, d, name):
-        return ud.revisions[name]
+        return ud.revision
    def gitpkgv_revision(self, ud, d, name):
        """
        Return a sortable revision number by counting commits in the history
        Based on gitpkgv.bblass in meta-openembedded
        """
-        rev = self._build_revision(ud, d, name)
+        rev = ud.revision
        localpath = ud.localpath
        rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev)
        if not os.path.exists(localpath):
            commits = None
        else:
            if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
-                from pipes import quote
                commits = bb.fetch2.runfetchcmd(
-                        "git rev-list %s -- | wc -l" % quote(rev),
+                        "git rev-list %s -- | wc -l" % shlex.quote(rev),
                        d, quiet=True).strip().lstrip('0')
                if commits:
                    open(rev_file, "w").write("%d\n" % int(commits))
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py
index f7f3af7212..5869e1b99b 100644
--- a/bitbake/lib/bb/fetch2/gitsm.py
+++ b/bitbake/lib/bb/fetch2/gitsm.py
@@ -62,36 +62,35 @@ class GitSM(Git):
            return modules
        # Collect the defined submodules, and their attributes
-        for name in ud.names:
+        try:
+            gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=workdir)
+        except:
+            # No submodules to update
+            gitmodules = ""
+        for m, md in parse_gitmodules(gitmodules).items():
            try:
-                gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
+                module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revision, md['path']), d, quiet=True, workdir=workdir)
            except:
-                # No submodules to update
+                # If the command fails, we don't have a valid file to check.  If it doesn't
+                # fail -- it still might be a failure, see next check...
+                module_hash = ""
+            if not module_hash:
+                logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m)
                continue
-            for m, md in parse_gitmodules(gitmodules).items():
+            submodules.append(m)
-                try:
+            paths[m] = md['path']
-                    module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
+            revision[m] = ud.revision
-                except:
+            uris[m] = md['url']
-                    # If the command fails, we don't have a valid file to check.  If it doesn't
+            subrevision[m] = module_hash.split()[2]
-                    # fail -- it still might be a failure, see next check...
-                    module_hash = ""
+            # Convert relative to absolute uri based on parent uri
+            if  uris[m].startswith('..') or uris[m].startswith('./'):
-                if not module_hash:
+                newud = copy.copy(ud)
-                    logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m)
+                newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
-                    continue
+                uris[m] = Git._get_repo_url(self, newud)
-                submodules.append(m)
-                paths[m] = md['path']
-                revision[m] = ud.revisions[name]
-                uris[m] = md['url']
-                subrevision[m] = module_hash.split()[2]
-                # Convert relative to absolute uri based on parent uri
-                if  uris[m].startswith('..') or uris[m].startswith('./'):
-                    newud = copy.copy(ud)
-                    newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
-                    uris[m] = Git._get_repo_url(self, newud)
        for module in submodules:
            # Translate the module url into a SRC_URI
@@ -123,7 +122,7 @@ class GitSM(Git):
            url += ";name=%s" % module
            url += ";subpath=%s" % module
            url += ";nobranch=1"
-            url += ";lfs=%s" % self._need_lfs(ud)
+            url += ";lfs=%s" % ("1" if self._need_lfs(ud) else "0")
            # Note that adding "user=" here to give credentials to the
            # submodule is not supported. Since using SRC_URI to give git://
            # URL a password is not supported, one have to use one of the
@@ -147,6 +146,22 @@ class GitSM(Git):
        return submodules != []
+    def call_process_submodules(self, ud, d, extra_check, subfunc):
+        # If we're using a shallow mirror tarball it needs to be
+        # unpacked temporarily so that we can examine the .gitmodules file
+        # Unpack even when ud.clonedir is not available,
+        # which may occur during a fast shallow clone
+        unpack = extra_check or not os.path.exists(ud.clonedir)
+        if ud.shallow and os.path.exists(ud.fullshallow) and unpack:
+            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
+            try:
+                runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
+                self.process_submodules(ud, tmpdir, subfunc, d)
+            finally:
+                shutil.rmtree(tmpdir)
+        else:
+            self.process_submodules(ud, ud.clonedir, subfunc, d)
    def need_update(self, ud, d):
        if Git.need_update(self, ud, d):
            return True
@@ -164,15 +179,7 @@ class GitSM(Git):
                logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e)))
                need_update_result = True
-        # If we're using a shallow mirror tarball it needs to be unpacked
+        self.call_process_submodules(ud, d, not os.path.exists(ud.clonedir), need_update_submodule)
-        # temporarily so that we can examine the .gitmodules file
-        if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir):
-            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
-            runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
-            self.process_submodules(ud, tmpdir, need_update_submodule, d)
-            shutil.rmtree(tmpdir)
-        else:
-            self.process_submodules(ud, ud.clonedir, need_update_submodule, d)
        if need_update_list:
            logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
@@ -195,16 +202,7 @@ class GitSM(Git):
                raise
        Git.download(self, ud, d)
+        self.call_process_submodules(ud, d, self.need_update(ud, d), download_submodule)
-        # If we're using a shallow mirror tarball it needs to be unpacked
-        # temporarily so that we can examine the .gitmodules file
-        if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
-            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
-            runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
-            self.process_submodules(ud, tmpdir, download_submodule, d)
-            shutil.rmtree(tmpdir)
-        else:
-            self.process_submodules(ud, ud.clonedir, download_submodule, d)
    def unpack(self, ud, destdir, d):
        def unpack_submodules(ud, url, module, modpath, workdir, d):
@@ -247,15 +245,27 @@ class GitSM(Git):
        ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
        if not ud.bareclone and ret:
-            # All submodules should already be downloaded and configured in the tree.  This simply
+            cmdprefix = ""
-            # sets up the configuration and checks out the files.  The main project config should
+            # Avoid LFS smudging (replacing the LFS pointers with the actual content) when LFS shouldn't be used but git-lfs is installed.
-            # remain unmodified, and no download from the internet should occur. As such, lfs smudge
+            if not self._need_lfs(ud):
-            # should also be skipped as these files were already smudged in the fetch stage if lfs
+                cmdprefix = "GIT_LFS_SKIP_SMUDGE=1 "
-            # was enabled.
+            runfetchcmd("%s%s submodule update --recursive --no-fetch" % (cmdprefix, ud.basecmd), d, quiet=True, workdir=ud.destdir)
-            runfetchcmd("GIT_LFS_SKIP_SMUDGE=1 %s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
+    def clean(self, ud, d):
+        def clean_submodule(ud, url, module, modpath, workdir, d):
+            url += ";bareclone=1;nobranch=1"
+            try:
+                newfetch = Fetch([url], d, cache=False)
+                newfetch.clean()
+            except Exception as e:
+                logger.warning('gitsm: submodule clean failed: %s %s' % (type(e).__name__, str(e)))
+        self.call_process_submodules(ud, d, True, clean_submodule)
+        # Clean top git dir
+        Git.clean(self, ud, d)
    def implicit_urldata(self, ud, d):
-        import shutil, subprocess, tempfile
+        import subprocess
        urldata = []
        def add_submodule(ud, url, module, modpath, workdir, d):
@@ -263,14 +273,6 @@ class GitSM(Git):
            newfetch = Fetch([url], d, cache=False)
            urldata.extend(newfetch.expanded_urldata())
-        # If we're using a shallow mirror tarball it needs to be unpacked
+        self.call_process_submodules(ud, d, ud.method.need_update(ud, d), add_submodule)
-        # temporarily so that we can examine the .gitmodules file
-        if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d):
-            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
-            subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True)
-            self.process_submodules(ud, tmpdir, add_submodule, d)
-            shutil.rmtree(tmpdir)
-        else:
-            self.process_submodules(ud, ud.clonedir, add_submodule, d)
        return urldata
diff --git a/bitbake/lib/bb/fetch2/gomod.py b/bitbake/lib/bb/fetch2/gomod.py
new file mode 100644
index 0000000000..53c1d8d115
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/gomod.py
@@ -0,0 +1,273 @@
+"""
+BitBake 'Fetch' implementation for Go modules
+The gomod/gomodgit fetchers are used to download Go modules to the module cache
+from a module proxy or directly from a version control repository.
+Example SRC_URI:
+SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..."
+SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;repo=go.googlesource.com/net;srcrev=..."
+Required SRC_URI parameters:
+- version
+    The version of the module.
+Optional SRC_URI parameters:
+- mod
+    Fetch and unpack the go.mod file only instead of the complete module.
+    The go command may need to download go.mod files for many different modules
+    when computing the build list, and go.mod files are much smaller than
+    module zip files.
+    The default is "0", set mod=1 for the go.mod file only.
+- sha256sum
+    The checksum of the module zip file, or the go.mod file in case of fetching
+    only the go.mod file. Alternatively, set the SRC_URI varible flag for
+    "module@version.sha256sum".
+- protocol
+    The method used when fetching directly from a version control repository.
+    The default is "https" for git.
+- repo
+    The URL when fetching directly from a version control repository. Required
+    when the URL is different from the module path.
+- srcrev
+    The revision identifier used when fetching directly from a version control
+    repository. Alternatively, set the SRCREV varible for "module@version".
+- subdir
+    The module subdirectory when fetching directly from a version control
+    repository. Required when the module is not located in the root of the
+    repository.
+Related variables:
+- GO_MOD_PROXY
+    The module proxy used by the fetcher.
+- GO_MOD_CACHE_DIR
+    The directory where the module cache is located.
+    This must match the exported GOMODCACHE variable for the go command to find
+    the downloaded modules.
+See the Go modules reference, https://go.dev/ref/mod, for more information
+about the module cache, module proxies and version control systems.
+"""
+import hashlib
+import os
+import re
+import shutil
+import subprocess
+import zipfile
+import bb
+from bb.fetch2 import FetchError
+from bb.fetch2 import MissingParameterError
+from bb.fetch2 import runfetchcmd
+from bb.fetch2 import subprocess_setup
+from bb.fetch2.git import Git
+from bb.fetch2.wget import Wget
+def escape(path):
+    """Escape capital letters using exclamation points."""
+    return re.sub(r'([A-Z])', lambda m: '!' + m.group(1).lower(), path)
+class GoMod(Wget):
+    """Class to fetch Go modules from a Go module proxy via wget"""
+    def supports(self, ud, d):
+        """Check to see if a given URL is for this fetcher."""
+        return ud.type == 'gomod'
+    def urldata_init(self, ud, d):
+        """Set up to download the module from the module proxy.
+        Set up to download the module zip file to the module cache directory
+        and unpack the go.mod file (unless downloading only the go.mod file):
+        cache/download/<module>/@v/<version>.zip: The module zip file.
+        cache/download/<module>/@v/<version>.mod: The go.mod file.
+        """
+        proxy = d.getVar('GO_MOD_PROXY') or 'proxy.golang.org'
+        moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
+        if 'version' not in ud.parm:
+            raise MissingParameterError('version', ud.url)
+        module = ud.host
+        if ud.path != '/':
+            module += ud.path
+        ud.parm['module'] = module
+        version = ud.parm['version']
+        # Set URL and filename for wget download
+        if ud.parm.get('mod', '0') == '1':
+            ext = '.mod'
+        else:
+            ext = '.zip'
+        path = escape(f"{module}/@v/{version}{ext}")
+        ud.url = bb.fetch2.encodeurl(
+            ('https', proxy, '/' + path, None, None, None))
+        ud.parm['downloadfilename'] =  f"{module.replace('/', '.')}@{version}{ext}"
+        # Set name for checksum verification
+        ud.parm['name'] = f"{module}@{version}"
+        # Set path for unpack
+        ud.parm['unpackpath'] = os.path.join(moddir, 'cache/download', path)
+        super().urldata_init(ud, d)
+    def unpack(self, ud, rootdir, d):
+        """Unpack the module in the module cache."""
+        # Unpack the module zip file or go.mod file
+        unpackpath = os.path.join(rootdir, ud.parm['unpackpath'])
+        unpackdir = os.path.dirname(unpackpath)
+        bb.utils.mkdirhier(unpackdir)
+        ud.unpack_tracer.unpack("file-copy", unpackdir)
+        cmd = f"cp {ud.localpath} {unpackpath}"
+        path = d.getVar('PATH')
+        if path:
+            cmd = f"PATH={path} {cmd}"
+        name = os.path.basename(unpackpath)
+        bb.note(f"Unpacking {name} to {unpackdir}/")
+        subprocess.check_call(cmd, shell=True, preexec_fn=subprocess_setup)
+        if name.endswith('.zip'):
+            # Unpack the go.mod file from the zip file
+            module = ud.parm['module']
+            name = name.rsplit('.', 1)[0] + '.mod'
+            bb.note(f"Unpacking {name} to {unpackdir}/")
+            with zipfile.ZipFile(ud.localpath) as zf:
+                with open(os.path.join(unpackdir, name), mode='wb') as mf:
+                    try:
+                        f = module + '@' + ud.parm['version'] + '/go.mod'
+                        shutil.copyfileobj(zf.open(f), mf)
+                    except KeyError:
+                        # If the module does not have a go.mod file, synthesize
+                        # one containing only a module statement.
+                        mf.write(f'module {module}\n'.encode())
+class GoModGit(Git):
+    """Class to fetch Go modules directly from a git repository"""
+    def supports(self, ud, d):
+        """Check to see if a given URL is for this fetcher."""
+        return ud.type == 'gomodgit'
+    def urldata_init(self, ud, d):
+        """Set up to download the module from the git repository.
+        Set up to download the git repository to the module cache directory and
+        unpack the module zip file and the go.mod file:
+        cache/vcs/<hash>:                         The bare git repository.
+        cache/download/<module>/@v/<version>.zip: The module zip file.
+        cache/download/<module>/@v/<version>.mod: The go.mod file.
+        """
+        moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
+        if 'version' not in ud.parm:
+            raise MissingParameterError('version', ud.url)
+        module = ud.host
+        if ud.path != '/':
+            module += ud.path
+        ud.parm['module'] = module
+        # Set host, path and srcrev for git download
+        if 'repo' in ud.parm:
+            repo = ud.parm['repo']
+            idx = repo.find('/')
+            if idx != -1:
+                ud.host = repo[:idx]
+                ud.path = repo[idx:]
+            else:
+                ud.host = repo
+                ud.path = ''
+        if 'protocol' not in ud.parm:
+            ud.parm['protocol'] = 'https'
+        ud.name = f"{module}@{ud.parm['version']}"
+        srcrev = d.getVar('SRCREV_' + ud.name)
+        if srcrev:
+            if 'srcrev' not in ud.parm:
+                ud.parm['srcrev'] = srcrev
+        else:
+            if 'srcrev' in ud.parm:
+                d.setVar('SRCREV_' + ud.name, ud.parm['srcrev'])
+        if 'branch' not in ud.parm:
+            ud.parm['nobranch'] = '1'
+        # Set subpath, subdir and bareclone for git unpack
+        if 'subdir' in ud.parm:
+            ud.parm['subpath'] = ud.parm['subdir']
+        key = f"git3:{ud.parm['protocol']}://{ud.host}{ud.path}".encode()
+        ud.parm['key'] = key
+        ud.parm['subdir'] = os.path.join(moddir, 'cache/vcs',
+                                         hashlib.sha256(key).hexdigest())
+        ud.parm['bareclone'] = '1'
+        super().urldata_init(ud, d)
+    def unpack(self, ud, rootdir, d):
+        """Unpack the module in the module cache."""
+        # Unpack the bare git repository
+        super().unpack(ud, rootdir, d)
+        moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
+        # Create the info file
+        module = ud.parm['module']
+        repodir = os.path.join(rootdir, ud.parm['subdir'])
+        with open(repodir + '.info', 'wb') as f:
+            f.write(ud.parm['key'])
+        # Unpack the go.mod file from the repository
+        unpackdir = os.path.join(rootdir, moddir, 'cache/download',
+                                 escape(module), '@v')
+        bb.utils.mkdirhier(unpackdir)
+        srcrev = ud.parm['srcrev']
+        version = ud.parm['version']
+        escaped_version = escape(version)
+        cmd = f"git ls-tree -r --name-only '{srcrev}'"
+        if 'subpath' in ud.parm:
+            cmd += f" '{ud.parm['subpath']}'"
+        files = runfetchcmd(cmd, d, workdir=repodir).split()
+        name = escaped_version + '.mod'
+        bb.note(f"Unpacking {name} to {unpackdir}/")
+        with open(os.path.join(unpackdir, name), mode='wb') as mf:
+            f = 'go.mod'
+            if 'subpath' in ud.parm:
+                f = os.path.join(ud.parm['subpath'], f)
+            if f in files:
+                cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
+                subprocess.check_call(cmd, stdout=mf, cwd=repodir,
+                                      preexec_fn=subprocess_setup)
+            else:
+                # If the module does not have a go.mod file, synthesize one
+                # containing only a module statement.
+                mf.write(f'module {module}\n'.encode())
+        # Synthesize the module zip file from the repository
+        name = escaped_version + '.zip'
+        bb.note(f"Unpacking {name} to {unpackdir}/")
+        with zipfile.ZipFile(os.path.join(unpackdir, name), mode='w') as zf:
+            prefix = module + '@' + version + '/'
+            for f in files:
+                cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
+                data = subprocess.check_output(cmd, cwd=repodir,
+                                               preexec_fn=subprocess_setup)
+                zf.writestr(prefix + f, data)
diff --git a/bitbake/lib/bb/fetch2/local.py b/bitbake/lib/bb/fetch2/local.py
index 7d7668110e..fda56a564e 100644
--- a/bitbake/lib/bb/fetch2/local.py
+++ b/bitbake/lib/bb/fetch2/local.py
@@ -29,11 +29,10 @@ class Local(FetchMethod):
    def urldata_init(self, ud, d):
        # We don't set localfile as for this fetcher the file is already local!
-        ud.decodedurl = urllib.parse.unquote(ud.url.split("://")[1].split(";")[0])
+        ud.basename = os.path.basename(ud.path)
-        ud.basename = os.path.basename(ud.decodedurl)
+        ud.basepath = ud.path
-        ud.basepath = ud.decodedurl
        ud.needdonestamp = False
-        if "*" in ud.decodedurl:
+        if "*" in ud.path:
            raise bb.fetch2.ParameterError("file:// urls using globbing are no longer supported. Please place the files in a directory and reference that instead.", ud.url)
        return
@@ -48,7 +47,7 @@ class Local(FetchMethod):
        Return the local filename of a given url assuming a successful fetch.
        """
        searched = []
-        path = urldata.decodedurl
+        path = urldata.path
        newpath = path
        if path[0] == "/":
            logger.debug2("Using absolute %s" % (path))
diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py
index 15f3f19bc8..e469d66768 100644
--- a/bitbake/lib/bb/fetch2/npm.py
+++ b/bitbake/lib/bb/fetch2/npm.py
@@ -42,11 +42,12 @@ from bb.utils import is_semver
 def npm_package(package):
    """Convert the npm package name to remove unsupported character"""
-    # Scoped package names (with the @) use the same naming convention
+    # For scoped package names ('@user/package') the '/' is replaced by a '-'.
-    # as the 'npm pack' command.
+    # This is similar to what 'npm pack' does, but 'npm pack' also strips the
+    # leading '@', which can lead to ambiguous package names.
    name = re.sub("/", "-", package)
    name = name.lower()
-    name = re.sub(r"[^\-a-z0-9]", "", name)
+    name = re.sub(r"[^\-a-z0-9@]", "", name)
    name = name.strip("-")
    return name
@@ -90,6 +91,12 @@ class NpmEnvironment(object):
        self.d = d
        self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1)
+        hn = self._home_npmrc(d)
+        if hn is not None:
+            with open(hn, 'r') as hnf:
+                self.user_config.write(hnf.read())
        for key, value in configs:
            self.user_config.write("%s=%s\n" % (key, value))
@@ -102,6 +109,15 @@ class NpmEnvironment(object):
        if self.user_config:
            self.user_config.close()
+    def _home_npmrc(self, d):
+        """Function to return user's HOME .npmrc file (or None if it doesn't exist)"""
+        home_npmrc_file = os.path.join(os.environ.get("HOME"), ".npmrc")
+        if d.getVar("BB_USE_HOME_NPMRC") == "1" and os.path.exists(home_npmrc_file):
+            bb.warn(f"BB_USE_HOME_NPMRC flag set and valid .npmrc detected - "\
+                    f"npm fetcher will use {home_npmrc_file}")
+            return home_npmrc_file
+        return None
    def run(self, cmd, args=None, configs=None, workdir=None):
        """Run npm command in a controlled environment"""
        with tempfile.TemporaryDirectory() as tmpdir:
@@ -165,7 +181,7 @@ class Npm(FetchMethod):
        # Using the 'downloadfilename' parameter as local filename
        # or the npm package name.
        if "downloadfilename" in ud.parm:
-            ud.localfile = npm_localfile(d.expand(ud.parm["downloadfilename"]))
+            ud.localfile = npm_localfile(ud.parm["downloadfilename"])
        else:
            ud.localfile = npm_localfile(ud.package, ud.version)
diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py
index ff5f8dc755..2f9599ee9e 100644
--- a/bitbake/lib/bb/fetch2/npmsw.py
+++ b/bitbake/lib/bb/fetch2/npmsw.py
@@ -37,38 +37,26 @@ def foreach_dependencies(shrinkwrap, callback=None, dev=False):
    """
        Run a callback for each dependencies of a shrinkwrap file.
        The callback is using the format:
-            callback(name, params, deptree)
+            callback(name, data, location)
        with:
            name = the package name (string)
-            params = the package parameters (dictionary)
+            data = the package data (dictionary)
-            destdir = the destination of the package (string)
+            location = the location of the package (string)
    """
-    # For handling old style dependencies entries in shinkwrap files
+    packages = shrinkwrap.get("packages")
-    def _walk_deps(deps, deptree):
+    if not packages:
-        for name in deps:
+        raise FetchError("Invalid shrinkwrap file format")
-            subtree = [*deptree, name]
-            _walk_deps(deps[name].get("dependencies", {}), subtree)
+    for location, data in packages.items():
-            if callback is not None:
+        # Skip empty main and local link target packages
-                if deps[name].get("dev", False) and not dev:
+        if not location.startswith('node_modules/'):
-                    continue
+            continue
-                elif deps[name].get("bundled", False):
+        elif not dev and data.get("dev", False):
-                    continue
+            continue
-                destsubdirs = [os.path.join("node_modules", dep) for dep in subtree]
+        elif data.get("inBundle", False):
-                destsuffix = os.path.join(*destsubdirs)
+            continue
-                callback(name, deps[name], destsuffix)
+        name = location.split('node_modules/')[-1]
+        callback(name, data, location)
-    # packages entry means new style shrinkwrap file, else use dependencies
-    packages = shrinkwrap.get("packages", None)
-    if packages is not None:
-        for package in packages:
-            if package != "":
-                name = package.split('node_modules/')[-1]
-                package_infos = packages.get(package, {})
-                if dev == False and package_infos.get("dev", False):
-                    continue
-                callback(name, package_infos, package)
-    else:
-        _walk_deps(shrinkwrap.get("dependencies", {}), [])
 class NpmShrinkWrap(FetchMethod):
    """Class to fetch all package from a shrinkwrap file"""
@@ -95,12 +83,18 @@ class NpmShrinkWrap(FetchMethod):
            extrapaths = []
            unpack = True
-            integrity = params.get("integrity", None)
+            integrity = params.get("integrity")
-            resolved = params.get("resolved", None)
+            resolved = params.get("resolved")
-            version = params.get("version", None)
+            version = params.get("version")
+            link = params.get("link", False)
+            # Handle link sources
+            if link:
+                localpath = resolved
+                unpack = False
            # Handle registry sources
-            if is_semver(version) and integrity:
+            elif version and is_semver(version) and integrity:
                # Handle duplicate dependencies without url
                if not resolved:
                    return
@@ -128,10 +122,10 @@ class NpmShrinkWrap(FetchMethod):
                extrapaths.append(resolvefile)
            # Handle http tarball sources
-            elif version.startswith("http") and integrity:
+            elif resolved.startswith("http") and integrity:
-                localfile = npm_localfile(os.path.basename(version))
+                localfile = npm_localfile(os.path.basename(resolved))
-                uri = URI(version)
+                uri = URI(resolved)
                uri.params["downloadfilename"] = localfile
                checksum_name, checksum_expected = npm_integrity(integrity)
@@ -141,28 +135,12 @@ class NpmShrinkWrap(FetchMethod):
                localpath = os.path.join(d.getVar("DL_DIR"), localfile)
-            # Handle local tarball and link sources
+            # Handle local tarball sources
-            elif version.startswith("file"):
+            elif resolved.startswith("file"):
-                localpath = version[5:]
+                localpath = resolved[5:]
-                if not version.endswith(".tgz"):
-                    unpack = False
            # Handle git sources
-            elif version.startswith(("git", "bitbucket","gist")) or (
+            elif resolved.startswith("git"):
-                not version.endswith((".tgz", ".tar", ".tar.gz"))
-                and not version.startswith((".", "@", "/"))
-                and "/" in version
-            ):
-                if version.startswith("github:"):
-                    version = "git+https://github.com/" + version[len("github:"):]
-                elif version.startswith("gist:"):
-                    version = "git+https://gist.github.com/" + version[len("gist:"):]
-                elif version.startswith("bitbucket:"):
-                    version = "git+https://bitbucket.org/" + version[len("bitbucket:"):]
-                elif version.startswith("gitlab:"):
-                    version = "git+https://gitlab.com/" + version[len("gitlab:"):]
-                elif not version.startswith(("git+","git:")):
-                    version = "git+https://github.com/" + version
                regex = re.compile(r"""
                    ^
                    git\+
@@ -174,16 +152,16 @@ class NpmShrinkWrap(FetchMethod):
                    $
                    """, re.VERBOSE)
-                match = regex.match(version)
+                match = regex.match(resolved)
                if not match:
-                    raise ParameterError("Invalid git url: %s" % version, ud.url)
+                    raise ParameterError("Invalid git url: %s" % resolved, ud.url)
                groups = match.groupdict()
                uri = URI("git://" + str(groups["url"]))
                uri.params["protocol"] = str(groups["protocol"])
                uri.params["rev"] = str(groups["rev"])
+                uri.params["nobranch"] = "1"
                uri.params["destsuffix"] = destsuffix
                url = str(uri)
@@ -268,7 +246,7 @@ class NpmShrinkWrap(FetchMethod):
    def unpack(self, ud, rootdir, d):
        """Unpack the downloaded dependencies"""
-        destdir = d.getVar("S")
+        destdir = rootdir
        destsuffix = ud.parm.get("destsuffix")
        if destsuffix:
            destdir = os.path.join(rootdir, destsuffix)
diff --git a/bitbake/lib/bb/fetch2/s3.py b/bitbake/lib/bb/fetch2/s3.py
index 6b8ffd5359..22c0538139 100644
--- a/bitbake/lib/bb/fetch2/s3.py
+++ b/bitbake/lib/bb/fetch2/s3.py
@@ -77,7 +77,7 @@ class S3(FetchMethod):
        else:
            ud.basename = os.path.basename(ud.path)
-        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+        ud.localfile = ud.basename
        ud.basecmd = d.getVar("FETCHCMD_s3") or "/usr/bin/env aws s3"
diff --git a/bitbake/lib/bb/fetch2/sftp.py b/bitbake/lib/bb/fetch2/sftp.py
index 7884cce949..bee71a0d0d 100644
--- a/bitbake/lib/bb/fetch2/sftp.py
+++ b/bitbake/lib/bb/fetch2/sftp.py
@@ -77,7 +77,7 @@ class SFTP(FetchMethod):
        else:
            ud.basename = os.path.basename(ud.path)
-        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+        ud.localfile = ud.basename
    def download(self, ud, d):
        """Fetch urls"""
diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py
index 0cbb2a6f25..2a0f2cb44b 100644
--- a/bitbake/lib/bb/fetch2/ssh.py
+++ b/bitbake/lib/bb/fetch2/ssh.py
@@ -73,8 +73,7 @@ class SSH(FetchMethod):
        path = m.group('path')
        path = urllib.parse.unquote(path)
        host = m.group('host')
-        urldata.localpath = os.path.join(d.getVar('DL_DIR'),
+        urldata.localfile = os.path.basename(os.path.normpath(path))
-                os.path.basename(os.path.normpath(path)))
    def download(self, urldata, d):
        dldir = d.getVar('DL_DIR')
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
index d40e4d2909..0852108e7d 100644
--- a/bitbake/lib/bb/fetch2/svn.py
+++ b/bitbake/lib/bb/fetch2/svn.py
@@ -210,3 +210,6 @@ class Svn(FetchMethod):
    def _build_revision(self, ud, d):
        return ud.revision
+    def supports_checksum(self, urldata):
+        return False
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index fbfa6938ac..7e43d3bc97 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -53,11 +53,6 @@ class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
 class Wget(FetchMethod):
    """Class to fetch urls via 'wget'"""
-    # CDNs like CloudFlare may do a 'browser integrity test' which can fail
-    # with the standard wget/urllib User-Agent, so pretend to be a modern
-    # browser.
-    user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0"
    def check_certs(self, d):
        """
        Should certificates be checked?
@@ -83,11 +78,11 @@ class Wget(FetchMethod):
        else:
            ud.basename = os.path.basename(ud.path)
-        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+        ud.localfile = ud.basename
        if not ud.localfile:
-            ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."))
+            ud.localfile = ud.host + ud.path.replace("/", ".")
-        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30"
+        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget --tries=2 --timeout=100"
        if ud.type == 'ftp' or ud.type == 'ftps':
            self.basecmd += " --passive-ftp"
@@ -101,16 +96,17 @@ class Wget(FetchMethod):
        logger.debug2("Fetching %s using command '%s'" % (ud.url, command))
        bb.fetch2.check_network_access(d, command, ud.url)
-        runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir)
+        runfetchcmd(command + ' --progress=dot --verbose', d, quiet, log=progresshandler, workdir=workdir)
    def download(self, ud, d):
        """Fetch urls"""
        fetchcmd = self.basecmd
-        localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) + ".tmp"
+        dldir = os.path.realpath(d.getVar("DL_DIR"))
+        localpath = os.path.join(dldir, ud.localfile) + ".tmp"
        bb.utils.mkdirhier(os.path.dirname(localpath))
-        fetchcmd += " -O %s" % shlex.quote(localpath)
+        fetchcmd += " --output-document=%s" % shlex.quote(localpath)
        if ud.user and ud.pswd:
            fetchcmd += " --auth-no-challenge"
@@ -126,14 +122,18 @@ class Wget(FetchMethod):
                fetchcmd += " --user=%s --password=%s" % (ud.user, ud.pswd)
        uri = ud.url.split(";")[0]
-        if os.path.exists(ud.localpath):
+        fetchcmd += " --continue --directory-prefix=%s '%s'" % (dldir, uri)
-            # file exists, but we didnt complete it.. trying again..
-            fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % uri)
-        else:
-            fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % uri)
        self._runwget(ud, d, fetchcmd, False)
+        # Sanity check since wget can pretend it succeed when it didn't
+        # Also, this used to happen if sourceforge sent us to the mirror page
+        if not os.path.exists(localpath):
+            raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, localpath), uri)
+        if os.path.getsize(localpath) == 0:
+            os.remove(localpath)
+            raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri)
        # Try and verify any checksum now, meaning if it isn't correct, we don't remove the
        # original file, which might be a race (imagine two recipes referencing the same
        # source, one with an incorrect checksum)
@@ -143,15 +143,6 @@ class Wget(FetchMethod):
        # Our lock prevents multiple writers but mirroring code may grab incomplete files
        os.rename(localpath, localpath[:-4])
-        # Sanity check since wget can pretend it succeed when it didn't
-        # Also, this used to happen if sourceforge sent us to the mirror page
-        if not os.path.exists(ud.localpath):
-            raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, ud.localpath), uri)
-        if os.path.getsize(ud.localpath) == 0:
-            os.remove(ud.localpath)
-            raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri)
        return True
    def checkstatus(self, fetch, ud, d, try_again=True):
@@ -243,7 +234,12 @@ class Wget(FetchMethod):
                        fetch.connection_cache.remove_connection(h.host, h.port)
                    raise urllib.error.URLError(err)
                else:
-                    r = h.getresponse()
+                    try:
+                        r = h.getresponse()
+                    except TimeoutError as e:
+                        if fetch.connection_cache:
+                            fetch.connection_cache.remove_connection(h.host, h.port)
+                        raise TimeoutError(e)
                # Pick apart the HTTPResponse object to get the addinfourl
                # object initialized properly.
@@ -304,13 +300,45 @@ class Wget(FetchMethod):
        class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
            """
-            urllib2.HTTPRedirectHandler resets the method to GET on redirect,
+            urllib2.HTTPRedirectHandler before 3.13 has two flaws:
-            when we want to follow redirects using the original method.
+            
+            It resets the method to GET on redirect when we want to follow
+            redirects using the original method (typically HEAD). This was fixed
+            in 759e8e7.
+            It also doesn't handle 308 (Permanent Redirect). This was fixed in
+            c379bc5.
+            Until we depend on Python 3.13 onwards, copy the redirect_request
+            method to fix these issues.
            """
            def redirect_request(self, req, fp, code, msg, headers, newurl):
-                newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
+                m = req.get_method()
-                newreq.get_method = req.get_method
+                if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD")
-                return newreq
+                    or code in (301, 302, 303) and m == "POST")):
+                    raise urllib.HTTPError(req.full_url, code, msg, headers, fp)
+                # Strictly (according to RFC 2616), 301 or 302 in response to
+                # a POST MUST NOT cause a redirection without confirmation
+                # from the user (of urllib.request, in this case).  In practice,
+                # essentially all clients do redirect in this case, so we do
+                # the same.
+                # Be conciliant with URIs containing a space.  This is mainly
+                # redundant with the more complete encoding done in http_error_302(),
+                # but it is kept for compatibility with other callers.
+                newurl = newurl.replace(' ', '%20')
+                CONTENT_HEADERS = ("content-length", "content-type")
+                newheaders = {k: v for k, v in req.headers.items()
+                            if k.lower() not in CONTENT_HEADERS}
+                return urllib.request.Request(newurl,
+                            method="HEAD" if m == "HEAD" else "GET",
+                            headers=newheaders,
+                            origin_req_host=req.origin_req_host,
+                            unverifiable=True)
+            http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302
        # We need to update the environment here as both the proxy and HTTPS
        # handlers need variables set. The proxy needs http_proxy and friends to
@@ -343,14 +371,14 @@ class Wget(FetchMethod):
            opener = urllib.request.build_opener(*handlers)
            try:
-                uri_base = ud.url.split(";")[0]
+                parts = urllib.parse.urlparse(ud.url.split(";")[0])
-                uri = "{}://{}{}".format(urllib.parse.urlparse(uri_base).scheme, ud.host, ud.path)
+                uri = "{}://{}{}".format(parts.scheme, parts.netloc, parts.path)
                r = urllib.request.Request(uri)
                r.get_method = lambda: "HEAD"
                # Some servers (FusionForge, as used on Alioth) require that the
                # optional Accept header is set.
                r.add_header("Accept", "*/*")
-                r.add_header("User-Agent", self.user_agent)
+                r.add_header("User-Agent", "bitbake/{}".format(bb.__version__))
                def add_basic_auth(login_str, request):
                    '''Adds Basic auth to http request, pass in login:password as string'''
                    import base64
@@ -370,7 +398,7 @@ class Wget(FetchMethod):
                except (FileNotFoundError, netrc.NetrcParseError):
                    pass
-                with opener.open(r, timeout=30) as response:
+                with opener.open(r, timeout=100) as response:
                    pass
            except (urllib.error.URLError, ConnectionResetError, TimeoutError) as e:
                if try_again:
@@ -457,7 +485,7 @@ class Wget(FetchMethod):
        f = tempfile.NamedTemporaryFile()
        with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
            fetchcmd = self.basecmd
-            fetchcmd += " -O " + f.name + " --user-agent='" + self.user_agent + "' '" + uri + "'"
+            fetchcmd += " --output-document=%s '%s'" % (f.name, uri)
            try:
                self._runwget(ud, d, fetchcmd, True, workdir=workdir)
                fetchresult = f.read()
@@ -617,13 +645,17 @@ class Wget(FetchMethod):
        sanity check to ensure same name and type.
        """
-        package = ud.path.split("/")[-1]
+        if 'downloadfilename' in ud.parm:
+            package = ud.parm['downloadfilename']
+        else:
+            package = ud.path.split("/")[-1]
        current_version = ['', d.getVar('PV'), '']
        """possible to have no version in pkg name, such as spectrum-fw"""
        if not re.search(r"\d+", package):
            current_version[1] = re.sub('_', '.', current_version[1])
            current_version[1] = re.sub('-', '.', current_version[1])
+            bb.debug(3, "latest_versionstring: no version found in %s" % package)
            return (current_version[1], '')
        package_regex = self._init_regexes(package, ud, d)