summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul Barker <pbarker@konsulko.com>2020-01-07 11:27:33 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-01-10 21:18:22 +0000
commit61e1b4136fb8ceb3785dcfdb22a3b80d062faaa6 (patch)
treed7f1abe452990c94c11f5e51cad9741f5f69e10a
parentf6c25f546b7baf823b21ccb29db8854112aeebd5 (diff)
downloadpoky-61e1b4136fb8ceb3785dcfdb22a3b80d062faaa6.tar.gz
archiver.bbclass: Add new mirror archiver mode
We define a new method of populating a source mirror using the archiver bbclass instead of simply copying the contents of the downloads directory. This allows the archiver features such as copyleft license filtering and recipe type filtering to be used when preparing a source mirror. This new archiver mode is selected by setting `ARCHIVE_MODE[src]` to 'mirror'. The source mirror mode can either be 'split' (default) or 'combined', controlled by `ARCHIVER_MODE[mirror]`. Additionally, sources can be excluded as needed by setting `ARCHIVER_MIRROR_EXCLUDE` to a list of URI prefixes. These options are described in more detail in the new entries in the header of archiver.bbclass. New oeqa selftest cases are added to cover the mirror archiver mode. (From OE-Core rev: 2c8b31ae0ab95a8b100e8bade23f51574e273c9a) Signed-off-by: Paul Barker <pbarker@konsulko.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/archiver.bbclass136
-rw-r--r--meta/lib/oeqa/selftest/cases/archiver.py59
2 files changed, 176 insertions, 19 deletions
diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass
index 7c46cff91f..013195df7d 100644
--- a/meta/classes/archiver.bbclass
+++ b/meta/classes/archiver.bbclass
@@ -2,25 +2,42 @@
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3# 3#
4# This bbclass is used for creating archive for: 4# This bbclass is used for creating archive for:
5# 1) original (or unpacked) source: ARCHIVER_MODE[src] = "original" 5# 1) original (or unpacked) source: ARCHIVER_MODE[src] = "original"
6# 2) patched source: ARCHIVER_MODE[src] = "patched" (default) 6# 2) patched source: ARCHIVER_MODE[src] = "patched" (default)
7# 3) configured source: ARCHIVER_MODE[src] = "configured" 7# 3) configured source: ARCHIVER_MODE[src] = "configured"
8# 4) The patches between do_unpack and do_patch: 8# 4) source mirror: ARCHIVE_MODE[src] = "mirror"
9# ARCHIVER_MODE[diff] = "1" 9# 5) The patches between do_unpack and do_patch:
10# And you can set the one that you'd like to exclude from the diff: 10# ARCHIVER_MODE[diff] = "1"
11# ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches" 11# And you can set the one that you'd like to exclude from the diff:
12# 5) The environment data, similar to 'bitbake -e recipe': 12# ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches"
13# ARCHIVER_MODE[dumpdata] = "1" 13# 6) The environment data, similar to 'bitbake -e recipe':
14# 6) The recipe (.bb and .inc): ARCHIVER_MODE[recipe] = "1" 14# ARCHIVER_MODE[dumpdata] = "1"
15# 7) Whether output the .src.rpm package: 15# 7) The recipe (.bb and .inc): ARCHIVER_MODE[recipe] = "1"
16# ARCHIVER_MODE[srpm] = "1" 16# 8) Whether output the .src.rpm package:
17# 8) Filter the license, the recipe whose license in 17# ARCHIVER_MODE[srpm] = "1"
18# COPYLEFT_LICENSE_INCLUDE will be included, and in 18# 9) Filter the license, the recipe whose license in
19# COPYLEFT_LICENSE_EXCLUDE will be excluded. 19# COPYLEFT_LICENSE_INCLUDE will be included, and in
20# COPYLEFT_LICENSE_INCLUDE = 'GPL* LGPL*' 20# COPYLEFT_LICENSE_EXCLUDE will be excluded.
21# COPYLEFT_LICENSE_EXCLUDE = 'CLOSED Proprietary' 21# COPYLEFT_LICENSE_INCLUDE = 'GPL* LGPL*'
22# 9) The recipe type that will be archived: 22# COPYLEFT_LICENSE_EXCLUDE = 'CLOSED Proprietary'
23# COPYLEFT_RECIPE_TYPES = 'target' 23# 10) The recipe type that will be archived:
24# COPYLEFT_RECIPE_TYPES = 'target'
25# 11) The source mirror mode:
26# ARCHIVER_MODE[mirror] = "split" (default): Sources are split into
27# per-recipe directories in a similar way to other archiver modes.
28# Post-processing may be required to produce a single mirror directory.
29# This does however allow inspection of duplicate sources and more
30# intelligent handling.
31# ARCHIVER_MODE[mirror] = "combined": All sources are placed into a single
32# directory suitable for direct use as a mirror. Duplicate sources are
33# ignored.
34# 12) Source mirror exclusions:
35# ARCHIVE_MIRROR_EXCLUDE is a list of prefixes to exclude from the mirror.
36# This may be used for sources which you are already publishing yourself
37# (e.g. if the URI starts with 'https://mysite.com/' and your mirror is
38# going to be published to the same site). It may also be used to exclude
39# local files (with the prefix 'file://') if these will be provided as part
40# of an archive of the layers themselves.
24# 41#
25 42
26# Create archive for all the recipe types 43# Create archive for all the recipe types
@@ -33,6 +50,7 @@ ARCHIVER_MODE[diff] ?= "0"
33ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches" 50ARCHIVER_MODE[diff-exclude] ?= ".pc autom4te.cache patches"
34ARCHIVER_MODE[dumpdata] ?= "0" 51ARCHIVER_MODE[dumpdata] ?= "0"
35ARCHIVER_MODE[recipe] ?= "0" 52ARCHIVER_MODE[recipe] ?= "0"
53ARCHIVER_MODE[mirror] ?= "split"
36 54
37DEPLOY_DIR_SRC ?= "${DEPLOY_DIR}/sources" 55DEPLOY_DIR_SRC ?= "${DEPLOY_DIR}/sources"
38ARCHIVER_TOPDIR ?= "${WORKDIR}/deploy-sources" 56ARCHIVER_TOPDIR ?= "${WORKDIR}/deploy-sources"
@@ -41,6 +59,10 @@ ARCHIVER_RPMTOPDIR ?= "${WORKDIR}/deploy-sources-rpm"
41ARCHIVER_RPMOUTDIR = "${ARCHIVER_RPMTOPDIR}/${TARGET_SYS}/${PF}/" 59ARCHIVER_RPMOUTDIR = "${ARCHIVER_RPMTOPDIR}/${TARGET_SYS}/${PF}/"
42ARCHIVER_WORKDIR = "${WORKDIR}/archiver-work/" 60ARCHIVER_WORKDIR = "${WORKDIR}/archiver-work/"
43 61
62# When producing a combined mirror directory, allow duplicates for the case
63# where multiple recipes use the same SRC_URI.
64ARCHIVER_COMBINED_MIRRORDIR = "${ARCHIVER_TOPDIR}/mirror"
65SSTATE_DUPWHITELIST += "${DEPLOY_DIR_SRC}/mirror"
44 66
45do_dumpdata[dirs] = "${ARCHIVER_OUTDIR}" 67do_dumpdata[dirs] = "${ARCHIVER_OUTDIR}"
46do_ar_recipe[dirs] = "${ARCHIVER_OUTDIR}" 68do_ar_recipe[dirs] = "${ARCHIVER_OUTDIR}"
@@ -106,6 +128,8 @@ python () {
106 elif hasTask("do_configure"): 128 elif hasTask("do_configure"):
107 d.appendVarFlag('do_ar_configured', 'depends', ' %s:do_configure' % pn) 129 d.appendVarFlag('do_ar_configured', 'depends', ' %s:do_configure' % pn)
108 d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_configured' % pn) 130 d.appendVarFlag('do_deploy_archives', 'depends', ' %s:do_ar_configured' % pn)
131 elif ar_src == "mirror":
132 d.appendVarFlag('do_deploy_archives', 'depends', '%s:do_ar_mirror' % pn)
109 133
110 elif ar_src: 134 elif ar_src:
111 bb.fatal("Invalid ARCHIVER_MODE[src]: %s" % ar_src) 135 bb.fatal("Invalid ARCHIVER_MODE[src]: %s" % ar_src)
@@ -281,6 +305,79 @@ python do_ar_configured() {
281 create_tarball(d, srcdir, 'configured', ar_outdir) 305 create_tarball(d, srcdir, 'configured', ar_outdir)
282} 306}
283 307
308python do_ar_mirror() {
309 import subprocess
310
311 src_uri = (d.getVar('SRC_URI') or '').split()
312 if len(src_uri) == 0:
313 return
314
315 dl_dir = d.getVar('DL_DIR')
316 mirror_exclusions = (d.getVar('ARCHIVER_MIRROR_EXCLUDE') or '').split()
317 mirror_mode = d.getVarFlag('ARCHIVER_MODE', 'mirror')
318 have_mirror_tarballs = d.getVar('BB_GENERATE_MIRROR_TARBALLS')
319
320 if mirror_mode == 'combined':
321 destdir = d.getVar('ARCHIVER_COMBINED_MIRRORDIR')
322 elif mirror_mode == 'split':
323 destdir = d.getVar('ARCHIVER_OUTDIR')
324 else:
325 bb.fatal('Invalid ARCHIVER_MODE[mirror]: %s' % (mirror_mode))
326
327 if not have_mirror_tarballs:
328 bb.fatal('Using `ARCHIVER_MODE[src] = "mirror"` depends on setting `BB_GENERATE_MIRROR_TARBALLS = "1"`')
329
330 def is_excluded(url):
331 for prefix in mirror_exclusions:
332 if url.startswith(prefix):
333 return True
334 return False
335
336 bb.note('Archiving the source as a mirror...')
337
338 bb.utils.mkdirhier(destdir)
339
340 fetcher = bb.fetch2.Fetch(src_uri, d)
341
342 for url in fetcher.urls:
343 if is_excluded(url):
344 bb.note('Skipping excluded url: %s' % (url))
345 continue
346
347 bb.note('Archiving url: %s' % (url))
348 ud = fetcher.ud[url]
349 ud.setup_localpath(d)
350 localpath = None
351
352 # Check for mirror tarballs first. We will archive the first mirror
353 # tarball that we find as it's assumed that we just need one.
354 for mirror_fname in ud.mirrortarballs:
355 mirror_path = os.path.join(dl_dir, mirror_fname)
356 if os.path.exists(mirror_path):
357 bb.note('Found mirror tarball: %s' % (mirror_path))
358 localpath = mirror_path
359 break
360
361 if len(ud.mirrortarballs) and not localpath:
362 bb.warn('Mirror tarballs are listed for a source but none are present.' \
363 'Falling back to original download.\n' \
364 'SRC_URI = %s' % (url))
365
366 # Check original download
367 if not localpath:
368 bb.note('Using original download: %s' % (ud.localpath))
369 localpath = ud.localpath
370
371 if not localpath or not os.path.exists(localpath):
372 bb.fatal('Original download is missing for a source.\n' \
373 'SRC_URI = %s' % (url))
374
375 # We now have an appropriate localpath
376 bb.note('Copying source mirror')
377 cmd = 'cp -fpPRH %s %s' % (localpath, destdir)
378 subprocess.check_call(cmd, shell=True)
379}
380
284def exclude_useless_paths(tarinfo): 381def exclude_useless_paths(tarinfo):
285 if tarinfo.isdir(): 382 if tarinfo.isdir():
286 if tarinfo.name.endswith('/temp') or tarinfo.name.endswith('/patches') or tarinfo.name.endswith('/.pc'): 383 if tarinfo.name.endswith('/temp') or tarinfo.name.endswith('/patches') or tarinfo.name.endswith('/.pc'):
@@ -483,6 +580,7 @@ addtask do_ar_original after do_unpack
483addtask do_unpack_and_patch after do_patch 580addtask do_unpack_and_patch after do_patch
484addtask do_ar_patched after do_unpack_and_patch 581addtask do_ar_patched after do_unpack_and_patch
485addtask do_ar_configured after do_unpack_and_patch 582addtask do_ar_configured after do_unpack_and_patch
583addtask do_ar_mirror after do_fetch
486addtask do_dumpdata 584addtask do_dumpdata
487addtask do_ar_recipe 585addtask do_ar_recipe
488addtask do_deploy_archives before do_build 586addtask do_deploy_archives before do_build
diff --git a/meta/lib/oeqa/selftest/cases/archiver.py b/meta/lib/oeqa/selftest/cases/archiver.py
index 6bd0e06ec4..fab2f56c39 100644
--- a/meta/lib/oeqa/selftest/cases/archiver.py
+++ b/meta/lib/oeqa/selftest/cases/archiver.py
@@ -195,3 +195,62 @@ class Archiver(OESelftestTestCase):
195 195
196 self._test_archiver_mode('patched', 'selftest-ed-1.14.1-r0-showdata.dump', 196 self._test_archiver_mode('patched', 'selftest-ed-1.14.1-r0-showdata.dump',
197 'ARCHIVER_MODE[dumpdata] = "1"\n') 197 'ARCHIVER_MODE[dumpdata] = "1"\n')
198
199 def test_archiver_mode_mirror(self):
200 """
201 Test that the archiver works with `ARCHIVER_MODE[src] = "mirror"`.
202 """
203
204 self._test_archiver_mode('mirror', 'ed-1.14.1.tar.lz',
205 'BB_GENERATE_MIRROR_TARBALLS = "1"\n')
206
207 def test_archiver_mode_mirror_excludes(self):
208 """
209 Test that the archiver works with `ARCHIVER_MODE[src] = "mirror"` and
210 correctly excludes an archive when its URL matches
211 `ARCHIVER_MIRROR_EXCLUDES`.
212 """
213
214 target='selftest-ed'
215 target_file_name = 'ed-1.14.1.tar.lz'
216
217 features = 'INHERIT += "archiver"\n'
218 features += 'ARCHIVER_MODE[src] = "mirror"\n'
219 features += 'BB_GENERATE_MIRROR_TARBALLS = "1"\n'
220 features += 'ARCHIVER_MIRROR_EXCLUDE = "${GNU_MIRROR}"\n'
221 self.write_config(features)
222
223 bitbake('-c clean %s' % (target))
224 bitbake('-c deploy_archives %s' % (target))
225
226 bb_vars = get_bb_vars(['DEPLOY_DIR_SRC', 'TARGET_SYS'])
227 glob_str = os.path.join(bb_vars['DEPLOY_DIR_SRC'], bb_vars['TARGET_SYS'], '%s-*' % (target))
228 glob_result = glob.glob(glob_str)
229 self.assertTrue(glob_result, 'Missing archiver directory for %s' % (target))
230
231 archive_path = os.path.join(glob_result[0], target_file_name)
232 self.assertFalse(os.path.exists(archive_path), 'Failed to exclude archive file %s' % (target_file_name))
233
234 def test_archiver_mode_mirror_combined(self):
235 """
236 Test that the archiver works with `ARCHIVER_MODE[src] = "mirror"`
237 and `ARCHIVER_MODE[mirror] = "combined"`. Archives for multiple recipes
238 should all end up in the 'mirror' directory.
239 """
240
241 features = 'INHERIT += "archiver"\n'
242 features += 'ARCHIVER_MODE[src] = "mirror"\n'
243 features += 'ARCHIVER_MODE[mirror] = "combined"\n'
244 features += 'BB_GENERATE_MIRROR_TARBALLS = "1"\n'
245 features += 'COPYLEFT_LICENSE_INCLUDE = "*"\n'
246 self.write_config(features)
247
248 for target in ['selftest-ed', 'selftest-hardlink']:
249 bitbake('-c clean %s' % (target))
250 bitbake('-c deploy_archives %s' % (target))
251
252 bb_vars = get_bb_vars(['DEPLOY_DIR_SRC'])
253 for target_file_name in ['ed-1.14.1.tar.lz', 'hello.c']:
254 glob_str = os.path.join(bb_vars['DEPLOY_DIR_SRC'], 'mirror', target_file_name)
255 glob_result = glob.glob(glob_str)
256 self.assertTrue(glob_result, 'Missing archive file %s' % (target_file_name))