From e1ba46109ea4be3d3b310abaf7f2da3c84a83930 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Fri, 20 Jul 2018 09:36:06 +0000 Subject: package: Call file to determine elf status in parallel This allows the calls to is_elf (which calls file) to happen in parallel allowing a speedup of do_package and do_populate_sysroot for native recipes. (From OE-Core rev: bbe0d3e26484f3f347262d40a8a9d415ce21fb43) Signed-off-by: Richard Purdie --- meta/classes/package.bbclass | 85 ++++++++++++++++++++++++++------------------ meta/classes/staging.bbclass | 4 +-- 2 files changed, 52 insertions(+), 37 deletions(-) (limited to 'meta/classes') diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass index f121acccab..81cb0c049c 100644 --- a/meta/classes/package.bbclass +++ b/meta/classes/package.bbclass @@ -949,6 +949,8 @@ python split_and_strip_files () { skipfiles = (d.getVar("INHIBIT_PACKAGE_STRIP_FILES") or "").split() if (d.getVar('INHIBIT_PACKAGE_STRIP') != '1' or \ d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'): + checkelf = {} + checkelflinks = {} for root, dirs, files in cpath.walk(dvar): for f in files: file = os.path.join(root, f) @@ -982,44 +984,57 @@ python split_and_strip_files () { # Check its an executable if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \ or ((file.startswith(libdir) or file.startswith(baselibdir)) and (".so" in f or ".node" in f)): - # If it's a symlink, and points to an ELF file, we capture the readlink target + if cpath.islink(file): - target = os.readlink(file) - if oe.package.is_elf(ltarget): - #bb.note("Sym: %s (%d)" % (ltarget, oe.package.is_elf(ltarget))) - symlinks[file] = target + checkelflinks[file] = ltarget continue + # Use a reference of device ID and inode number to identify files + file_reference = "%d_%d" % (s.st_dev, s.st_ino) + checkelf[file] = (file, file_reference) + + results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelflinks.values(), d) + results_map = {} + for (ltarget, elf_file) in results: + results_map[ltarget] = elf_file + for file in checkelflinks: + ltarget = checkelflinks[file] + # If it's a symlink, and points to an ELF file, we capture the readlink target + if results_map[ltarget]: + target = os.readlink(file) + #bb.note("Sym: %s (%d)" % (ltarget, results_map[ltarget])) + symlinks[file] = target + + results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelf.keys(), d) + for (file, elf_file) in results: + # It's a file (or hardlink), not a link + # ...but is it ELF, and is it already stripped? + if elf_file & 1: + if elf_file & 2: + if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split(): + bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn)) + else: + msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn) + package_qa_handle_error("already-stripped", msg, d) + continue - # It's a file (or hardlink), not a link - # ...but is it ELF, and is it already stripped? - elf_file = oe.package.is_elf(file) - if elf_file & 1: - if elf_file & 2: - if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split(): - bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn)) - else: - msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn) - package_qa_handle_error("already-stripped", msg, d) - continue - - # At this point we have an unstripped elf file. We need to: - # a) Make sure any file we strip is not hardlinked to anything else outside this tree - # b) Only strip any hardlinked file once (no races) - # c) Track any hardlinks between files so that we can reconstruct matching debug file hardlinks - - # Use a reference of device ID and inode number to identify files - file_reference = "%d_%d" % (s.st_dev, s.st_ino) - if file_reference in inodes: - os.unlink(file) - os.link(inodes[file_reference][0], file) - inodes[file_reference].append(file) - else: - inodes[file_reference] = [file] - # break hardlink - bb.utils.copyfile(file, file) - elffiles[file] = elf_file - # Modified the file so clear the cache - cpath.updatecache(file) + # At this point we have an unstripped elf file. We need to: + # a) Make sure any file we strip is not hardlinked to anything else outside this tree + # b) Only strip any hardlinked file once (no races) + # c) Track any hardlinks between files so that we can reconstruct matching debug file hardlinks + + # Use a reference of device ID and inode number to identify files + file_reference = checkelf[file] + if file_reference in inodes: + os.unlink(file) + os.link(inodes[file_reference][0], file) + inodes[file_reference].append(file) + else: + inodes[file_reference] = [file] + # break hardlink + bb.utils.copyfile(file, file) + elffiles[file] = elf_file + # Modified the file so clear the cache + cpath.updatecache(file) # # First lets process debug splitting diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass index 939042eb44..41df883495 100644 --- a/meta/classes/staging.bbclass +++ b/meta/classes/staging.bbclass @@ -70,7 +70,7 @@ sysroot_stage_all() { python sysroot_strip () { inhibit_sysroot = d.getVar('INHIBIT_SYSROOT_STRIP') if inhibit_sysroot and oe.types.boolean(inhibit_sysroot): - return 0 + return dstdir = d.getVar('SYSROOT_DESTDIR') pn = d.getVar('PN') @@ -79,7 +79,7 @@ python sysroot_strip () { qa_already_stripped = 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split() strip_cmd = d.getVar("STRIP") - oe.package.strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, + oe.package.strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d, qa_already_stripped=qa_already_stripped) } -- cgit v1.2.3-54-g00ecf