vk-gl-cts: don't require networking to configure

The CMakeLists in this package go and download a number of packages at
configure time, which is bad practise for us.

Instead, use a script to parse the fetching tool and generate SRC_URI
fragments that can be included in the recipe. This refresh_srcuri task
will need to be reran on upgrades to ensure that it is up to date: the
fragment will warn if the version doesn't match and devtool will do that
automatically.

Signed-off-by: Ross Burton <ross.burton@arm.com>
Signed-off-by: Khem Raj <raj.khem@gmail.com>

6 files changed, 205 insertions, 55 deletions

diff --git a/meta-oe/recipes-graphics/vk-gl-cts/files/generate-srcuri.py b/meta-oe/recipes-graphics/vk-gl-cts/files/generate-srcuri.py
new file mode 100755
index 0000000000..c2756b592c
--- /dev/null
+++ b/meta-oe/recipes-graphics/vk-gl-cts/files/generate-srcuri.py
@@ -0,0 +1,131 @@
+#! /usr/bin/env python3
+
+import json
+import re
+import os
+import subprocess
+import sys
+import types
+import urllib.parse
+
+
+def resolve_commit(repo, ref):
+    # If it looks like a SHA, just return that
+    if re.match(r"[a-z0-9]{40}", ref):
+        return ref
+
+    # Otherwise it's probably a tag name so resolve it
+    cmd = ("git", "ls-remote", "--tags", "--exit-code", repo, ref)
+    ret = subprocess.run(cmd, check=True, text=True, capture_output=True)
+    sha = ret.stdout.split(maxsplit=1)[0]
+    assert len(sha) == 40
+    return sha
+
+
+def transform_git(repo_url, repo_ref, destination):
+    parts = urllib.parse.urlparse(repo_url)
+    protocol = parts.scheme
+    parts = parts._replace(scheme="git")
+    url = urllib.parse.urlunparse(parts)
+    # Resolve the commit reference to a SHA
+    sha = resolve_commit(repo_url, repo_ref)
+
+    return url + f";protocol={protocol};nobranch=1;destsuffix={destination};rev={sha}"
+
+
+def load_module(filename):
+    import importlib.util
+
+    spec = importlib.util.spec_from_file_location("fetchmodule", filename)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def convert_fetch(basedir):
+    """
+    Convert the external/fetch_sources.py data
+    """
+    fetch = load_module(os.path.join(basedir, "fetch_sources.py"))
+    lines = []
+    for p in fetch.PACKAGES:
+        if isinstance(p, fetch.SourcePackage):
+            # Ignore these as so far we can use the system copies
+            pass
+        elif isinstance(p, fetch.SourceFile):
+            dest = "/".join(["git/external", p.baseDir, p.extractDir])
+            url = f"{p.url};subdir={dest};sha256sum={p.checksum}"
+            lines.append(f"    {url} \\")
+        elif isinstance(p, fetch.GitRepo):
+            dest = "/".join(["git/external", p.baseDir, p.extractDir])
+            url = transform_git(p.httpsUrl, p.revision, dest)
+            lines.append(f"    {url} \\")
+        else:
+            assert f"Unexpected {p=}"
+    return lines
+
+
+def convert_knowngood(basedir, destination):
+    """
+    Convert the """
+    filename = os.path.join(basedir, "vulkan-validationlayers/src/scripts/known_good.json")
+    try:
+        with open(filename) as fp:
+            data = json.load(fp, object_hook=lambda x: types.SimpleNamespace(**x))
+    except FileNotFoundError:
+        return []
+
+    lines = []
+    for repo in data.repos:
+        # Skip repositories that are not needed on Linux (TODO: assumes linux target)
+        if hasattr(repo, "build_platforms") and repo.build_platforms != "linux":
+            continue
+
+        # Switch the URL to use git: and save the original protocol
+        parts = urllib.parse.urlparse(repo.url)
+        protocol = parts.scheme
+        parts = parts._replace(scheme="git")
+        url = urllib.parse.urlunparse(parts)
+        # Resolve the commit reference to a SHA
+        sha = resolve_commit(repo.url, repo.commit)
+
+        destsuffix = destination + "/" + repo.sub_dir
+
+        url += f";protocol={protocol};nobranch=1;destsuffix={destsuffix};rev={sha}"
+        lines.append(f"    {url} \\")
+    return lines
+
+
+def main():
+    pv = sys.argv[1]
+    basedir = sys.argv[2]
+
+    print("""
+#
+# Generated by generate-srcuri.py, don't update manually")
+#
+
+RECIPE_UPGRADE_EXTRA_TASKS += "do_refresh_srcuri"
+
+python __anonymous() {
+    if d.getVar("PV") != "%s":
+        bb.warn("-sources.inc out of date, run refresh_srcuri task")
+}
+""" % (pv))
+
+    print('SRC_URI += " \\')
+    lines = convert_fetch(basedir)
+    print("\n".join(lines))
+    print('"')
+
+    #lines = convert_knowngood(sys.argv[1], "git/external/validation")
+    #if lines:
+    #    print('SRC_URI += " \\')
+    #    print("\n".join(lines))
+    #    print('"')
+    #else:
+    #    print("# Re-run")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/meta-oe/recipes-graphics/vk-gl-cts/khronos-cts.inc b/meta-oe/recipes-graphics/vk-gl-cts/khronos-cts.inc
index 0b347cf77e..02af1c6ed0 100644
--- a/meta-oe/recipes-graphics/vk-gl-cts/khronos-cts.inc
+++ b/meta-oe/recipes-graphics/vk-gl-cts/khronos-cts.inc
@@ -1,21 +1,16 @@
 LICENSE = "Apache-2.0"
 LIC_FILES_CHKSUM = "file://LICENSE;md5=3b83ef96387f14655fc854ddc3c6bd57"
 
-SRC_URI = "\
-        git://github.com/KhronosGroup/VK-GL-CTS.git;protocol=https;name=vk-gl-cts;nobranch=1 \
-        git://github.com/google/amber;protocol=https;destsuffix=git/external/amber/src;name=amber;nobranch=1 \
-        git://github.com/KhronosGroup/glslang.git;protocol=https;destsuffix=git/external/glslang/src;name=glslang;nobranch=1 \
-        git://github.com/KhronosGroup/SPIRV-Headers.git;protocol=https;destsuffix=git/external/spirv-headers/src;name=spirv-headers;nobranch=1 \
-        git://github.com/KhronosGroup/SPIRV-Tools.git;protocol=https;destsuffix=git/external/spirv-tools/src;name=spirv-tools;nobranch=1 \
-        git://github.com/open-source-parsers/jsoncpp.git;protocol=https;destsuffix=git/external/jsoncpp/src;name=jsoncpp;nobranch=1 \
-        git://github.com/KhronosGroup/Vulkan-Docs.git;protocol=https;destsuffix=git/external/vulkan-docs/src;name=vulkan-docs;nobranch=1 \
-        git://github.com/KhronosGroup/Vulkan-ValidationLayers.git;protocol=https;destsuffix=git/external/vulkan-validationlayers/src;name=vulkan-validationlayers;nobranch=1 \
-        git://github.com/Igalia/ESExtractor.git;protocol=https;destsuffix=git/external/ESExtractor/src;name=ESExtractor;nobranch=1 \
-        git://github.com/Igalia/vk_video_samples.git;protocol=https;destsuffix=git/external/nvidia-video-samples/src;name=video-parser;nobranch=1 \
-        https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h;subdir=git/external/renderdoc/src;name=renderdoc \
-"
-
-SRCREV_FORMAT = "vk-gl-cts_amber_glslang_spirv-headers_spirv-tools_jsoncpp_video-parser_vulkan-docs_vulkan-validationlayers"
+SRC_URI = "git://github.com/KhronosGroup/VK-GL-CTS.git;protocol=https;name=vk-gl-cts;nobranch=1 \
+           file://0001-cmake-Define-WAYLAND_SCANNER-and-WAYLAND_PROTOCOLS_D.patch \
+           file://0001-use-library-sonames-for-linking.patch \
+           file://generate-srcuri.py \
+           "
+
+SRC_URI:append:libc-musl = "file://fix-musl.patch"
+SRC_URI:append:toolchain-clang = "file://fix-clang-private-operator.patch"
+
+SRCREV_FORMAT = "vk-gl-cts"
 
 S = "${WORKDIR}/git"
 
@@ -26,22 +21,10 @@ UPSTREAM_CHECK_GITTAGREGEX = "${BPN}-(?P<pver>\d+(\.\d+)+)"
 ANY_OF_DISTRO_FEATURES += "opengl vulkan"
 
 DEPENDS += "python3-lxml-native libpng zlib virtual/libgles2 qemu-native"
-
-SRC_URI += " \
-            file://0001-cmake-Define-WAYLAND_SCANNER-and-WAYLAND_PROTOCOLS_D.patch \
-            file://0001-use-library-sonames-for-linking.patch \
-"
-
-SRC_URI:append:libc-musl = "\
-        file://fix-musl.patch \
-"
 DEPENDS:append:libc-musl = " libexecinfo"
 
-SRC_URI:append:toolchain-clang = "\
-        file://fix-clang-private-operator.patch \
-"
-
 EXTRA_OECMAKE += "-DAMBER_DISABLE_WERROR=ON \
+                  -DUPDATE_DEPS_DIR=${S}/external/validation/ \
                   -DWAYLAND_SCANNER=${STAGING_BINDIR_NATIVE}/wayland-scanner \
                   -DWAYLAND_PROTOCOLS_DIR=${STAGING_DATADIR}/wayland-protocols"
 
@@ -78,6 +61,15 @@ FILES:${PN} += "${CTSDIR}"
 # error: implicit instantiation of undefined template 'std::char_traits<unsigned int>'
 TOOLCHAIN = "gcc"
 
-# Validation-layers requires access during configure as it fetches validation-headers
-# and bunch of other packages from khronos github
-do_configure[network] = "1"
+# Prototype task to refresh the generated SRC_URI entries by parsing
+# the files in the source tree and writing a BPN-sources.inc file.
+do_refresh_srcuri() {
+    ${UNPACKDIR}/generate-srcuri.py ${PV} ${S}/external/ \
+        > ${THISDIR}/${BPN}-sources.inc
+    # Don't convert ${S}/external/vulkan-validationlayers/src/scripts/known_good.json as we
+    # currently build without validation
+}
+
+addtask refresh_srcuri after do_patch
+do_refresh_srcuri[network] = "1"
+do_refresh_srcuri[nostamp] = "1"
diff --git a/meta-oe/recipes-graphics/vk-gl-cts/opengl-es-cts-sources.inc b/meta-oe/recipes-graphics/vk-gl-cts/opengl-es-cts-sources.inc
new file mode 100644
index 0000000000..9752c2d7d7
--- /dev/null
+++ b/meta-oe/recipes-graphics/vk-gl-cts/opengl-es-cts-sources.inc
@@ -0,0 +1,23 @@
+
+#
+# Generated by generate-srcuri.py, don't update manually")
+#
+
+RECIPE_UPGRADE_EXTRA_TASKS += "do_refresh_srcuri"
+
+python __anonymous() {
+    if d.getVar("PV") != "3.2.11.0":
+        bb.warn("-sources.inc out of date, run refresh_srcuri task")
+}
+
+SRC_URI += " \
+    https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h;subdir=git/external/renderdoc/src;sha256sum=e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e \
+    git://github.com/KhronosGroup/SPIRV-Tools.git;protocol=https;nobranch=1;destsuffix=git/external/spirv-tools/src;rev=148c97f6876e427efd76d2328122c3075eab4b8f \
+    git://github.com/KhronosGroup/glslang.git;protocol=https;nobranch=1;destsuffix=git/external/glslang/src;rev=4da479aa6afa43e5a2ce4c4148c572a03123faf3 \
+    git://github.com/KhronosGroup/SPIRV-Headers.git;protocol=https;nobranch=1;destsuffix=git/external/spirv-headers/src;rev=ff2afc3afc48dff4eec2a10f0212402a80708e38 \
+    git://github.com/KhronosGroup/Vulkan-Docs.git;protocol=https;nobranch=1;destsuffix=git/external/vulkan-docs/src;rev=ed4ba0242beb89a1795d6084709fa9e713559c94 \
+    git://github.com/KhronosGroup/Vulkan-ValidationLayers.git;protocol=https;nobranch=1;destsuffix=git/external/vulkan-validationlayers/src;rev=f589bc456545fbab97caf49380b102b8aafe1f40 \
+    git://github.com/google/amber.git;protocol=https;nobranch=1;destsuffix=git/external/amber/src;rev=0f003c2785489f59cd01bb2440fcf303149100f2 \
+    git://github.com/open-source-parsers/jsoncpp.git;protocol=https;nobranch=1;destsuffix=git/external/jsoncpp/src;rev=9059f5cad030ba11d37818847443a53918c327b1 \
+    git://github.com/Igalia/vk_video_samples.git;protocol=https;nobranch=1;destsuffix=git/external/nvidia-video-samples/src;rev=6821adf11eb4f84a2168264b954c170d03237699 \
+"
diff --git a/meta-oe/recipes-graphics/vk-gl-cts/opengl-es-cts_3.2.11.0.bb b/meta-oe/recipes-graphics/vk-gl-cts/opengl-es-cts_3.2.11.0.bb
index 1a745c6cc1..9d07076951 100644
--- a/meta-oe/recipes-graphics/vk-gl-cts/opengl-es-cts_3.2.11.0.bb
+++ b/meta-oe/recipes-graphics/vk-gl-cts/opengl-es-cts_3.2.11.0.bb
@@ -1,19 +1,10 @@
 DESCRIPTION = "OpenGL CTS"
 
 require khronos-cts.inc
-# opengl-es-cts-3.2.11.0
+
 SRCREV_vk-gl-cts = "66956d195169596472e956e3aebf2df8e3bd960d"
-SRCREV_amber = "0f003c2785489f59cd01bb2440fcf303149100f2"
-SRCREV_glslang = "4da479aa6afa43e5a2ce4c4148c572a03123faf3"
-SRCREV_spirv-headers = "ff2afc3afc48dff4eec2a10f0212402a80708e38"
-SRCREV_spirv-tools = "148c97f6876e427efd76d2328122c3075eab4b8f"
-SRCREV_ESExtractor = "ce5d7ebcf0ebb0d78385ee4cc34653eb6764bfc4"
-# Not yet needed
-SRCREV_jsoncpp = "9059f5cad030ba11d37818847443a53918c327b1"
-SRCREV_vulkan-docs = "ed4ba0242beb89a1795d6084709fa9e713559c94"
-SRCREV_vulkan-validationlayers = "a92629196a4fed15e59c74aa965dd47bd5ece3b7"    
-SRCREV_video-parser = "6821adf11eb4f84a2168264b954c170d03237699"
-SRC_URI[renderdoc.sha256sum] = "e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e"
+
+require opengl-es-cts-sources.inc
 
 EXTRA_OECMAKE += "-DSELECTED_BUILD_TARGETS="cts-runner deqp-egl deqp-gles2 deqp-gles3 deqp-gles31 deqp-gl-shared de-internal-tests glcts""
 
diff --git a/meta-oe/recipes-graphics/vk-gl-cts/vulkan-cts-sources.inc b/meta-oe/recipes-graphics/vk-gl-cts/vulkan-cts-sources.inc
new file mode 100644
index 0000000000..1f54282fcb
--- /dev/null
+++ b/meta-oe/recipes-graphics/vk-gl-cts/vulkan-cts-sources.inc
@@ -0,0 +1,23 @@
+
+#
+# Generated by generate-srcuri.py, don't update manually")
+#
+
+RECIPE_UPGRADE_EXTRA_TASKS += "do_refresh_srcuri"
+
+python __anonymous() {
+    if d.getVar("PV") != "1.3.9.2":
+        bb.warn("-sources.inc out of date, run refresh_srcuri task")
+}
+
+SRC_URI += " \
+    https://raw.githubusercontent.com/baldurk/renderdoc/v1.1/renderdoc/api/app/renderdoc_app.h;subdir=git/external/renderdoc/src;sha256sum=e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e \
+    git://github.com/KhronosGroup/SPIRV-Tools.git;protocol=https;nobranch=1;destsuffix=git/external/spirv-tools/src;rev=4c7e1fa5c3d988cca0e626d359d30b117b9c2822 \
+    git://github.com/KhronosGroup/glslang.git;protocol=https;nobranch=1;destsuffix=git/external/glslang/src;rev=2b19bf7e1bc0b60cf2fe9d33e5ba6b37dfc1cc83 \
+    git://github.com/KhronosGroup/SPIRV-Headers.git;protocol=https;nobranch=1;destsuffix=git/external/spirv-headers/src;rev=db5a00f8cebe81146cafabf89019674a3c4bf03d \
+    git://github.com/KhronosGroup/Vulkan-Docs.git;protocol=https;nobranch=1;destsuffix=git/external/vulkan-docs/src;rev=7bb606eb87cde1d34f65f36f4d4c6f2c78f072c8 \
+    git://github.com/KhronosGroup/Vulkan-ValidationLayers.git;protocol=https;nobranch=1;destsuffix=git/external/vulkan-validationlayers/src;rev=f589bc456545fbab97caf49380b102b8aafe1f40 \
+    git://github.com/google/amber.git;protocol=https;nobranch=1;destsuffix=git/external/amber/src;rev=0f003c2785489f59cd01bb2440fcf303149100f2 \
+    git://github.com/open-source-parsers/jsoncpp.git;protocol=https;nobranch=1;destsuffix=git/external/jsoncpp/src;rev=9059f5cad030ba11d37818847443a53918c327b1 \
+    git://github.com/Igalia/vk_video_samples.git;protocol=https;nobranch=1;destsuffix=git/external/nvidia-video-samples/src;rev=6821adf11eb4f84a2168264b954c170d03237699 \
+"
diff --git a/meta-oe/recipes-graphics/vk-gl-cts/vulkan-cts_1.3.9.2.bb b/meta-oe/recipes-graphics/vk-gl-cts/vulkan-cts_1.3.9.2.bb
index bffec29e4d..6c0a07b873 100644
--- a/meta-oe/recipes-graphics/vk-gl-cts/vulkan-cts_1.3.9.2.bb
+++ b/meta-oe/recipes-graphics/vk-gl-cts/vulkan-cts_1.3.9.2.bb
@@ -2,19 +2,9 @@ DESCRIPTION = "Vulkan CTS"
 
 require khronos-cts.inc
 
-# vulkan-cts-1.3.9.2
 SRCREV_vk-gl-cts = "24c1b1498ba4f05777f47541968ffe686265c645"
-SRCREV_amber = "0f003c2785489f59cd01bb2440fcf303149100f2"
-SRCREV_glslang = "2b19bf7e1bc0b60cf2fe9d33e5ba6b37dfc1cc83"
-SRCREV_spirv-headers = "db5a00f8cebe81146cafabf89019674a3c4bf03d"
-SRCREV_spirv-tools = "4c7e1fa5c3d988cca0e626d359d30b117b9c2822"
-SRCREV_jsoncpp = "9059f5cad030ba11d37818847443a53918c327b1"
-SRCREV_vulkan-docs = "7bb606eb87cde1d34f65f36f4d4c6f2c78f072c8"
-SRCREV_vulkan-validationlayers = "a92629196a4fed15e59c74aa965dd47bd5ece3b7"
-SRC_URI[renderdoc.sha256sum] = "e7b5f0aa5b1b0eadc63a1c624c0ca7f5af133aa857d6a4271b0ef3d0bdb6868e"
-# Not yet needed
-SRCREV_ESExtractor = "75ffcaf55bb069f7a23764194742d2fb78c7f71f"
-SRCREV_video-parser = "6821adf11eb4f84a2168264b954c170d03237699"
+
+require vulkan-cts-sources.inc
 
 # Workaround an optimization bug that breaks createMeshShaderMiscTestsEXT
 OECMAKE_CXX_FLAGS:remove:toolchain-gcc = "-O2"