diff options
author | Robert Yang <liezhi.yang@windriver.com> | 2023-12-21 23:31:49 -0800 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2023-12-23 08:46:01 +0000 |
commit | fdef56a596b937d099545aee5d0ff6fd4abc4c84 (patch) | |
tree | f3aaf9a6a31cf02c97a1bc4cc3d5cbf9ecdec8ed /meta/classes/archiver.bbclass | |
parent | 0d9de34ac6772c0b32ef8a54c2367d14691fb6e4 (diff) | |
download | poky-fdef56a596b937d099545aee5d0ff6fd4abc4c84.tar.gz |
archiver.bbclass: Drop tarfile module to improve performance
* The tarfile module doesn't support xz options or environment varible
XZ_DEFAULTS, this makes do_ar_patched incrediblely slow when the file is
large, for example, chromium-x11 is about 3GB:
- "bitbake chromium-x11 -car_patched" hasn't been done after 3 hours on my
host, I checked the partial tar.xz file is only 1.5GB, so maybe more than 6
hours is required to complete the task.
- Now only less than 4 minutes is needed on the same host.
* Need add xz to HOSTTOOLS when archiver.bbclass is enabled and compression is xz.
(From OE-Core rev: 6548354f049b173e8d443bc547d35c9d9fc05259)
Signed-off-by: Robert Yang <liezhi.yang@windriver.com>
Signed-off-by: Alexandre Belloni <alexandre.belloni@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/classes/archiver.bbclass')
-rw-r--r-- | meta/classes/archiver.bbclass | 28 |
1 files changed, 16 insertions, 12 deletions
diff --git a/meta/classes/archiver.bbclass b/meta/classes/archiver.bbclass index 80a69cf31d..2d0bbfbd42 100644 --- a/meta/classes/archiver.bbclass +++ b/meta/classes/archiver.bbclass | |||
@@ -401,19 +401,11 @@ python do_ar_mirror() { | |||
401 | subprocess.check_call(cmd, shell=True) | 401 | subprocess.check_call(cmd, shell=True) |
402 | } | 402 | } |
403 | 403 | ||
404 | def exclude_useless_paths(tarinfo): | ||
405 | if tarinfo.isdir(): | ||
406 | if tarinfo.name.endswith('/temp') or tarinfo.name.endswith('/patches') or tarinfo.name.endswith('/.pc'): | ||
407 | return None | ||
408 | elif tarinfo.name == 'temp' or tarinfo.name == 'patches' or tarinfo.name == '.pc': | ||
409 | return None | ||
410 | return tarinfo | ||
411 | |||
412 | def create_tarball(d, srcdir, suffix, ar_outdir): | 404 | def create_tarball(d, srcdir, suffix, ar_outdir): |
413 | """ | 405 | """ |
414 | create the tarball from srcdir | 406 | create the tarball from srcdir |
415 | """ | 407 | """ |
416 | import tarfile | 408 | import subprocess |
417 | 409 | ||
418 | # Make sure we are only creating a single tarball for gcc sources | 410 | # Make sure we are only creating a single tarball for gcc sources |
419 | if (d.getVar('SRC_URI') == ""): | 411 | if (d.getVar('SRC_URI') == ""): |
@@ -425,6 +417,16 @@ def create_tarball(d, srcdir, suffix, ar_outdir): | |||
425 | srcdir = os.path.realpath(srcdir) | 417 | srcdir = os.path.realpath(srcdir) |
426 | 418 | ||
427 | compression_method = d.getVarFlag('ARCHIVER_MODE', 'compression') | 419 | compression_method = d.getVarFlag('ARCHIVER_MODE', 'compression') |
420 | if compression_method == "xz": | ||
421 | compression_cmd = "xz %s" % d.getVar('XZ_DEFAULTS') | ||
422 | # To keep compatibility with ARCHIVER_MODE[compression] | ||
423 | elif compression_method == "gz": | ||
424 | compression_cmd = "gzip" | ||
425 | elif compression_method == "bz2": | ||
426 | compression_cmd = "bzip2" | ||
427 | else: | ||
428 | bb.fatal("Unsupported compression_method: %s" % compression_method) | ||
429 | |||
428 | bb.utils.mkdirhier(ar_outdir) | 430 | bb.utils.mkdirhier(ar_outdir) |
429 | if suffix: | 431 | if suffix: |
430 | filename = '%s-%s.tar.%s' % (d.getVar('PF'), suffix, compression_method) | 432 | filename = '%s-%s.tar.%s' % (d.getVar('PF'), suffix, compression_method) |
@@ -433,9 +435,11 @@ def create_tarball(d, srcdir, suffix, ar_outdir): | |||
433 | tarname = os.path.join(ar_outdir, filename) | 435 | tarname = os.path.join(ar_outdir, filename) |
434 | 436 | ||
435 | bb.note('Creating %s' % tarname) | 437 | bb.note('Creating %s' % tarname) |
436 | tar = tarfile.open(tarname, 'w:%s' % compression_method) | 438 | dirname = os.path.dirname(srcdir) |
437 | tar.add(srcdir, arcname=os.path.basename(srcdir), filter=exclude_useless_paths) | 439 | basename = os.path.basename(srcdir) |
438 | tar.close() | 440 | exclude = "--exclude=temp --exclude=patches --exclude='.pc'" |
441 | tar_cmd = "tar %s -cf - %s | %s > %s" % (exclude, basename, compression_cmd, tarname) | ||
442 | subprocess.check_call(tar_cmd, cwd=dirname, shell=True) | ||
439 | 443 | ||
440 | # creating .diff.gz between source.orig and source | 444 | # creating .diff.gz between source.orig and source |
441 | def create_diff_gz(d, src_orig, src, ar_outdir): | 445 | def create_diff_gz(d, src_orig, src, ar_outdir): |