archiver.bbclass: Add new mirror archiver mode

We define a new method of populating a source mirror using the archiver
bbclass instead of simply copying the contents of the downloads
directory. This allows the archiver features such as copyleft license
filtering and recipe type filtering to be used when preparing a source
mirror.

This new archiver mode is selected by setting `ARCHIVE_MODE[src]` to
'mirror'.

The source mirror mode can either be 'split' (default) or 'combined',
controlled by `ARCHIVER_MODE[mirror]`. Additionally, sources can be
excluded as needed by setting `ARCHIVER_MIRROR_EXCLUDE` to a list of
URI prefixes. These options are described in more detail in the new
entries in the header of archiver.bbclass.

New oeqa selftest cases are added to cover the mirror archiver mode.

(From OE-Core rev: 2c8b31ae0ab95a8b100e8bade23f51574e273c9a)

Signed-off-by: Paul Barker <pbarker@konsulko.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

2 files changed, 176 insertions, 19 deletions

diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass
index 7c46cff91f..013195df7d 100644
--- a/meta/classes/archiver.bbclass
+++ b/meta/classes/archiver.bbclass
@@ -2,25 +2,42 @@
 # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
 #
 # This bbclass is used for creating archive for:
-# 1) original (or unpacked) source: ARCHIVER_MODE[src] = "original"
-# 2) patched source: ARCHIVER_MODE[src] = "patched" (default)
-# 3) configured source: ARCHIVER_MODE[src] = "configured"
-# 4) The patches between do_unpack and do_patch:
-#    ARCHIVER_MODE[diff] = "1"
-#    And you can set the one that you'd like to exclude from the diff:
-#    ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches"
-# 5) The environment data, similar to 'bitbake -e recipe':
-#    ARCHIVER_MODE[dumpdata] = "1"
-# 6) The recipe (.bb and .inc): ARCHIVER_MODE[recipe] = "1"
-# 7) Whether output the .src.rpm package:
-#    ARCHIVER_MODE[srpm] = "1"
-# 8) Filter the license, the recipe whose license in
-#    COPYLEFT_LICENSE_INCLUDE will be included, and in
-#    COPYLEFT_LICENSE_EXCLUDE will be excluded.
-#    COPYLEFT_LICENSE_INCLUDE = 'GPL* LGPL*'
-#    COPYLEFT_LICENSE_EXCLUDE = 'CLOSED Proprietary'
-# 9) The recipe type that will be archived:
-#    COPYLEFT_RECIPE_TYPES = 'target'
+#  1) original (or unpacked) source: ARCHIVER_MODE[src] = "original"
+#  2) patched source: ARCHIVER_MODE[src] = "patched" (default)
+#  3) configured source: ARCHIVER_MODE[src] = "configured"
+#  4) source mirror: ARCHIVE_MODE[src] = "mirror"
+#  5) The patches between do_unpack and do_patch:
+#     ARCHIVER_MODE[diff] = "1"
+#     And you can set the one that you'd like to exclude from the diff:
+#     ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches"
+#  6) The environment data, similar to 'bitbake -e recipe':
+#     ARCHIVER_MODE[dumpdata] = "1"
+#  7) The recipe (.bb and .inc): ARCHIVER_MODE[recipe] = "1"
+#  8) Whether output the .src.rpm package:
+#     ARCHIVER_MODE[srpm] = "1"
+#  9) Filter the license, the recipe whose license in
+#     COPYLEFT_LICENSE_INCLUDE will be included, and in
+#     COPYLEFT_LICENSE_EXCLUDE will be excluded.
+#     COPYLEFT_LICENSE_INCLUDE = 'GPL* LGPL*'
+#     COPYLEFT_LICENSE_EXCLUDE = 'CLOSED Proprietary'
+# 10) The recipe type that will be archived:
+#     COPYLEFT_RECIPE_TYPES = 'target'
+# 11) The source mirror mode:
+#     ARCHIVER_MODE[mirror] = "split" (default): Sources are split into
+#     per-recipe directories in a similar way to other archiver modes.
+#     Post-processing may be required to produce a single mirror directory.
+#     This does however allow inspection of duplicate sources and more
+#     intelligent handling.
+#     ARCHIVER_MODE[mirror] = "combined": All sources are placed into a single
+#     directory suitable for direct use as a mirror. Duplicate sources are
+#     ignored.
+# 12) Source mirror exclusions:
+#     ARCHIVE_MIRROR_EXCLUDE is a list of prefixes to exclude from the mirror.
+#     This may be used for sources which you are already publishing yourself
+#     (e.g. if the URI starts with 'https://mysite.com/' and your mirror is
+#     going to be published to the same site). It may also be used to exclude
+#     local files (with the prefix 'file://') if these will be provided as part
+#     of an archive of the layers themselves.
 #
 
 # Create archive for all the recipe types
@@ -33,6 +50,7 @@ ARCHIVER_MODE[diff] ?= "0"
 ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches"
 ARCHIVER_MODE[dumpdata] ?= "0"
 ARCHIVER_MODE[recipe] ?= "0"
+ARCHIVER_MODE[mirror] ?= "split"
 
 DEPLOY_DIR_SRC ?= "${DEPLOY_DIR}/sources"
 ARCHIVER_TOPDIR ?= "${WORKDIR}/deploy-sources"
@@ -41,6 +59,10 @@ ARCHIVER_RPMTOPDIR ?= "${WORKDIR}/deploy-sources-rpm"
 ARCHIVER_RPMOUTDIR = "${ARCHIVER_RPMTOPDIR}/${TARGET_SYS}/${PF}/"
 ARCHIVER_WORKDIR = "${WORKDIR}/archiver-work/"
 
+# When producing a combined mirror directory, allow duplicates for the case
+# where multiple recipes use the same SRC_URI.
+ARCHIVER_COMBINED_MIRRORDIR = "${ARCHIVER_TOPDIR}/mirror"
+SSTATE_DUPWHITELIST += "${DEPLOY_DIR_SRC}/mirror"
 
 do_dumpdata[dirs] = "${ARCHIVER_OUTDIR}"
 do_ar_recipe[dirs] = "${ARCHIVER_OUTDIR}"
@@ -106,6 +128,8 @@ python () {
         elif hasTask("do_configure"):
             d.appendVarFlag('do_ar_configured', 'depends', ' %s:do_configure' % pn)
         d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_configured' % pn)
+    elif ar_src == "mirror":
+        d.appendVarFlag('do_deploy_archives', 'depends', '%s:do_ar_mirror' % pn)
 
     elif ar_src:
         bb.fatal("Invalid ARCHIVER_MODE[src]: %s" % ar_src)
@@ -281,6 +305,79 @@ python do_ar_configured() {
         create_tarball(d, srcdir, 'configured', ar_outdir)
 }
 
+python do_ar_mirror() {
+    import subprocess
+
+    src_uri = (d.getVar('SRC_URI') or '').split()
+    if len(src_uri) == 0:
+        return
+
+    dl_dir = d.getVar('DL_DIR')
+    mirror_exclusions = (d.getVar('ARCHIVER_MIRROR_EXCLUDE') or '').split()
+    mirror_mode = d.getVarFlag('ARCHIVER_MODE', 'mirror')
+    have_mirror_tarballs = d.getVar('BB_GENERATE_MIRROR_TARBALLS')
+
+    if mirror_mode == 'combined':
+        destdir = d.getVar('ARCHIVER_COMBINED_MIRRORDIR')
+    elif mirror_mode == 'split':
+        destdir = d.getVar('ARCHIVER_OUTDIR')
+    else:
+        bb.fatal('Invalid ARCHIVER_MODE[mirror]: %s' % (mirror_mode))
+
+    if not have_mirror_tarballs:
+        bb.fatal('Using `ARCHIVER_MODE[src] = "mirror"` depends on setting `BB_GENERATE_MIRROR_TARBALLS = "1"`')
+
+    def is_excluded(url):
+        for prefix in mirror_exclusions:
+            if url.startswith(prefix):
+                return True
+        return False
+
+    bb.note('Archiving the source as a mirror...')
+
+    bb.utils.mkdirhier(destdir)
+
+    fetcher = bb.fetch2.Fetch(src_uri, d)
+
+    for url in fetcher.urls:
+        if is_excluded(url):
+            bb.note('Skipping excluded url: %s' % (url))
+            continue
+
+        bb.note('Archiving url: %s' % (url))
+        ud = fetcher.ud[url]
+        ud.setup_localpath(d)
+        localpath = None
+
+        # Check for mirror tarballs first. We will archive the first mirror
+        # tarball that we find as it's assumed that we just need one.
+        for mirror_fname in ud.mirrortarballs:
+            mirror_path = os.path.join(dl_dir, mirror_fname)
+            if os.path.exists(mirror_path):
+                bb.note('Found mirror tarball: %s' % (mirror_path))
+                localpath = mirror_path
+                break
+
+        if len(ud.mirrortarballs) and not localpath:
+            bb.warn('Mirror tarballs are listed for a source but none are present.' \
+                    'Falling back to original download.\n' \
+                    'SRC_URI = %s' % (url))
+
+        # Check original download
+        if not localpath:
+            bb.note('Using original download: %s' % (ud.localpath))
+            localpath = ud.localpath
+
+        if not localpath or not os.path.exists(localpath):
+            bb.fatal('Original download is missing for a source.\n' \
+                        'SRC_URI = %s' % (url))
+
+        # We now have an appropriate localpath
+        bb.note('Copying source mirror')
+        cmd = 'cp -fpPRH %s %s' % (localpath, destdir)
+        subprocess.check_call(cmd, shell=True)
+}
+
 def exclude_useless_paths(tarinfo):
     if tarinfo.isdir():
         if tarinfo.name.endswith('/temp') or tarinfo.name.endswith('/patches') or tarinfo.name.endswith('/.pc'):
@@ -483,6 +580,7 @@ addtask do_ar_original after do_unpack
 addtask do_unpack_and_patch after do_patch
 addtask do_ar_patched after do_unpack_and_patch
 addtask do_ar_configured after do_unpack_and_patch
+addtask do_ar_mirror after do_fetch
 addtask do_dumpdata
 addtask do_ar_recipe
 addtask do_deploy_archives before do_build
diff --git a/meta/lib/oeqa/selftest/cases/archiver.py b/meta/lib/oeqa/selftest/cases/archiver.py
index 6bd0e06ec4..fab2f56c39 100644
--- a/meta/lib/oeqa/selftest/cases/archiver.py
+++ b/meta/lib/oeqa/selftest/cases/archiver.py
@@ -195,3 +195,62 @@ class Archiver(OESelftestTestCase):
 
         self._test_archiver_mode('patched', 'selftest-ed-1.14.1-r0-showdata.dump',
                                  'ARCHIVER_MODE[dumpdata] = "1"\n')
+
+    def test_archiver_mode_mirror(self):
+        """
+        Test that the archiver works with `ARCHIVER_MODE[src] = "mirror"`.
+        """
+
+        self._test_archiver_mode('mirror', 'ed-1.14.1.tar.lz',
+                                 'BB_GENERATE_MIRROR_TARBALLS = "1"\n')
+
+    def test_archiver_mode_mirror_excludes(self):
+        """
+        Test that the archiver works with `ARCHIVER_MODE[src] = "mirror"` and
+        correctly excludes an archive when its URL matches
+        `ARCHIVER_MIRROR_EXCLUDES`.
+        """
+
+        target='selftest-ed'
+        target_file_name = 'ed-1.14.1.tar.lz'
+
+        features = 'INHERIT += "archiver"\n'
+        features += 'ARCHIVER_MODE[src] = "mirror"\n'
+        features += 'BB_GENERATE_MIRROR_TARBALLS = "1"\n'
+        features += 'ARCHIVER_MIRROR_EXCLUDE = "${GNU_MIRROR}"\n'
+        self.write_config(features)
+
+        bitbake('-c clean %s' % (target))
+        bitbake('-c deploy_archives %s' % (target))
+
+        bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS'])
+        glob_str = os.path.join(bb_vars['DEPLOY_DIR_SRC'], bb_vars['TARGET_SYS'], '%s-*' % (target))
+        glob_result = glob.glob(glob_str)
+        self.assertTrue(glob_result, 'Missing archiver directory for %s' % (target))
+
+        archive_path = os.path.join(glob_result[0], target_file_name)
+        self.assertFalse(os.path.exists(archive_path), 'Failed to exclude archive file %s' % (target_file_name))
+
+    def test_archiver_mode_mirror_combined(self):
+        """
+        Test that the archiver works with `ARCHIVER_MODE[src] = "mirror"`
+        and `ARCHIVER_MODE[mirror] = "combined"`. Archives for multiple recipes
+        should all end up in the 'mirror' directory.
+        """
+
+        features = 'INHERIT += "archiver"\n'
+        features += 'ARCHIVER_MODE[src] = "mirror"\n'
+        features += 'ARCHIVER_MODE[mirror] = "combined"\n'
+        features += 'BB_GENERATE_MIRROR_TARBALLS = "1"\n'
+        features += 'COPYLEFT_LICENSE_INCLUDE = "*"\n'
+        self.write_config(features)
+
+        for target in ['selftest-ed', 'selftest-hardlink']:
+            bitbake('-c clean %s' % (target))
+            bitbake('-c deploy_archives %s' % (target))
+
+        bb_vars = get_bb_vars(['DEPLOY_DIR_SRC'])
+        for target_file_name in ['ed-1.14.1.tar.lz', 'hello.c']:
+            glob_str = os.path.join(bb_vars['DEPLOY_DIR_SRC'], 'mirror', target_file_name)
+            glob_result = glob.glob(glob_str)
+            self.assertTrue(glob_result, 'Missing archive file %s' % (target_file_name))