classes: go-vendor: Add go-vendor class

(From OE-Core rev: d61bdf392e10140671ca56f2a2b0dc824be8ab80)

Signed-off-by: Lukas Funke <lukas.funke@weidmueller.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 200 insertions, 0 deletions

diff --git a/meta/classes/go-vendor.bbclass b/meta/classes/go-vendor.bbclass
new file mode 100644
index 0000000000..5b017b0b9d
--- /dev/null
+++ b/meta/classes/go-vendor.bbclass
@@ -0,0 +1,200 @@
+#
+# Copyright 2023 (C) Weidmueller GmbH & Co KG
+# Author: Lukas Funke <lukas.funke@weidmueller.com>
+#
+# Handle Go vendor support for offline builds
+#
+# When importing Go modules, Go downloads the imported modules using
+# a network (proxy) connection ahead of the compile stage. This contradicts 
+# the yocto build concept of fetching every source ahead of build-time
+# and supporting offline builds.
+#
+# To support offline builds, we use Go 'vendoring': module dependencies are 
+# downloaded during the fetch-phase and unpacked into the modules 'vendor'
+# folder. Additionally a manifest file is generated for the 'vendor' folder
+# 
+
+inherit go-mod
+
+def go_src_uri(repo, version, path=None, subdir=None, \
+                vcs='git', replaces=None, pathmajor=None):
+
+    destsuffix = "git/src/import/vendor.fetch"
+    module_path = repo if not path else path
+
+    src_uri = "{}://{};name={}".format(vcs, repo, module_path.replace('/', '.'))
+    src_uri += ";destsuffix={}/{}@{}".format(destsuffix, repo, version)
+
+    if vcs == "git":
+        src_uri += ";nobranch=1;protocol=https"
+
+    src_uri += ";go_module_path={}".format(module_path)
+
+    if replaces:
+        src_uri += ";go_module_replacement={}".format(replaces)
+    if subdir:
+        src_uri += ";go_subdir={}".format(subdir)
+    if pathmajor:
+        src_uri += ";go_pathmajor={}".format(pathmajor)
+    src_uri += ";is_go_dependency=1"
+
+    return src_uri
+
+python do_vendor_unlink() {
+
+    # We unlink
+
+    go_import = d.getVar('GO_IMPORT')
+    source_dir = d.getVar('S')
+    linkname = os.path.join(source_dir, *['src', go_import, 'vendor'])
+
+    os.unlink(linkname)
+}
+
+addtask vendor_unlink before do_install after do_compile
+
+python do_go_vendor() {
+    import shutil
+
+    src_uri = (d.getVar('SRC_URI') or "").split()
+
+    if len(src_uri) == 0:
+        bb.error("SRC_URI is empty")
+        return
+
+    default_destsuffix = "git/src/import/vendor.fetch"
+    fetcher = bb.fetch2.Fetch(src_uri, d)
+    go_import = d.getVar('GO_IMPORT')
+    source_dir = d.getVar('S')
+
+    linkname = os.path.join(source_dir, *['src', go_import, 'vendor'])
+    vendor_dir = os.path.join(source_dir, *['src', 'import', 'vendor'])
+    import_dir = os.path.join(source_dir, *['src', 'import', 'vendor.fetch'])
+
+    if os.path.exists(vendor_dir):
+        # Nothing to do except re-establish link to actual vendor folder
+        if not os.path.exists(linkname):
+            os.symlink(vendor_dir, linkname)
+        return
+
+    bb.utils.mkdirhier(vendor_dir)
+
+    modules = {}
+
+    for url in fetcher.urls:
+        srcuri = fetcher.ud[url].host + fetcher.ud[url].path
+
+        # Skip non Go module src uris
+        if not fetcher.ud[url].parm.get('is_go_dependency'):
+            continue
+
+        destsuffix = fetcher.ud[url].parm.get('destsuffix')
+        # We derive the module repo / version in the following manner (exmaple):
+        # 
+        # destsuffix = git/src/import/vendor.fetch/github.com/foo/bar@v1.2.3
+        # p = github.com/foo/bar@v1.2.3
+        # repo = github.com/foo/bar
+        # version = v1.2.3
+
+        p = destsuffix[len(default_destsuffix)+1:]
+        repo, version = p.split('@')
+
+        module_path = fetcher.ud[url].parm.get('go_module_path')
+
+        subdir = fetcher.ud[url].parm.get('go_subdir')
+        subdir = None if not subdir else subdir
+
+        pathMajor = fetcher.ud[url].parm.get('go_pathmajor')
+        pathMajor = None if not pathMajor else pathMajor.strip('/')
+
+        if not repo in modules:
+            modules[repo] =   { "version": version,
+                                "repo_path": os.path.join(import_dir, p),
+                                "module_path": module_path,
+                                "subdir": subdir,
+                                "pathMajor": pathMajor }
+
+    for module_key in sorted(modules):
+
+        # only take the version which is explicitly listed
+        # as a dependency in the go.mod
+        module = modules[module_key]
+        module_path = module['module_path']
+        rootdir = module['repo_path']
+        subdir = module['subdir']
+        pathMajor = module['pathMajor']
+
+        src = rootdir
+
+        if subdir:
+            src = os.path.join(rootdir, subdir)
+
+        # If the module is released at major version 2 or higher, the module
+        # path must end with a major version suffix like /v2.
+        # This may or may not be part of the subdirectory name
+        #
+        # https://go.dev/ref/mod#modules-overview
+        if pathMajor:
+            tmp = os.path.join(src, pathMajor)
+            # source directory including major version path may or may not exist
+            if os.path.exists(tmp):
+                src = tmp
+
+        dst = os.path.join(vendor_dir, module_path)
+
+        bb.debug(1, "cp %s --> %s" % (src, dst))
+        shutil.copytree(src, dst, symlinks=True, \
+            ignore=shutil.ignore_patterns(".git", \
+                                            "vendor", \
+                                            "*._test.go"))
+
+        # If the root directory has a LICENSE file but not the subdir
+        # we copy the root license to the sub module since the license
+        # applies to all modules in the repository
+        # see https://go.dev/ref/mod#vcs-license
+        if subdir:
+            rootdirLicese = os.path.join(rootdir, "LICENSE")
+            subdirLicense = os.path.join(src, "LICENSE")
+
+            if not os.path.exists(subdir) and \
+                os.path.exists(rootdirLicese):
+                shutil.copy2(rootdirLicese, subdirLicense)
+
+    # Copy vendor manifest
+    modules_txt_src = os.path.join(d.getVar('WORKDIR'), "modules.txt")
+    bb.debug(1, "cp %s --> %s" % (modules_txt_src, vendor_dir))
+    shutil.copy2(modules_txt_src, vendor_dir)
+
+    # Clean up vendor dir
+    # We only require the modules in the modules_txt file
+    fetched_paths = set([os.path.relpath(x[0], vendor_dir) for x in os.walk(vendor_dir)])
+
+    # Remove toplevel dir
+    fetched_paths.remove('.')
+
+    vendored_paths = set()
+    with open(modules_txt_src) as f:
+        for line in f:
+            if not line.startswith("#"):
+                line = line.strip()
+                vendored_paths.add(line)
+
+                # Add toplevel dirs into vendored dir, as we want to keep them
+                topdir = os.path.dirname(line)
+                while len(topdir):
+                    if not topdir in vendored_paths:
+                        vendored_paths.add(topdir)
+
+                    topdir = os.path.dirname(topdir)
+
+    for path in fetched_paths:
+        if path not in vendored_paths:
+            realpath = os.path.join(vendor_dir, path)
+            if os.path.exists(realpath):
+                shutil.rmtree(realpath)
+
+    # Create a symlink the the actual directory
+    os.symlink(vendor_dir, linkname)
+}
+
+addtask go_vendor before do_patch after do_unpack