summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2018-07-20 09:36:06 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-07-24 11:52:27 +0100
commite1ba46109ea4be3d3b310abaf7f2da3c84a83930 (patch)
tree6ecddb381be06b2016eee1ede16c5b706c348120
parent069a1c4a1520914234dc58b506663fcfd80433e3 (diff)
downloadpoky-e1ba46109ea4be3d3b310abaf7f2da3c84a83930.tar.gz
package: Call file to determine elf status in parallel
This allows the calls to is_elf (which calls file) to happen in parallel allowing a speedup of do_package and do_populate_sysroot for native recipes. (From OE-Core rev: bbe0d3e26484f3f347262d40a8a9d415ce21fb43) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/package.bbclass85
-rw-r--r--meta/classes/staging.bbclass4
-rw-r--r--meta/lib/oe/package.py18
3 files changed, 64 insertions, 43 deletions
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index f121acccab..81cb0c049c 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -949,6 +949,8 @@ python split_and_strip_files () {
949 skipfiles = (d.getVar("INHIBIT_PACKAGE_STRIP_FILES") or "").split() 949 skipfiles = (d.getVar("INHIBIT_PACKAGE_STRIP_FILES") or "").split()
950 if (d.getVar('INHIBIT_PACKAGE_STRIP') != '1' or \ 950 if (d.getVar('INHIBIT_PACKAGE_STRIP') != '1' or \
951 d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'): 951 d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT') != '1'):
952 checkelf = {}
953 checkelflinks = {}
952 for root, dirs, files in cpath.walk(dvar): 954 for root, dirs, files in cpath.walk(dvar):
953 for f in files: 955 for f in files:
954 file = os.path.join(root, f) 956 file = os.path.join(root, f)
@@ -982,44 +984,57 @@ python split_and_strip_files () {
982 # Check its an executable 984 # Check its an executable
983 if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \ 985 if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
984 or ((file.startswith(libdir) or file.startswith(baselibdir)) and (".so" in f or ".node" in f)): 986 or ((file.startswith(libdir) or file.startswith(baselibdir)) and (".so" in f or ".node" in f)):
985 # If it's a symlink, and points to an ELF file, we capture the readlink target 987
986 if cpath.islink(file): 988 if cpath.islink(file):
987 target = os.readlink(file) 989 checkelflinks[file] = ltarget
988 if oe.package.is_elf(ltarget):
989 #bb.note("Sym: %s (%d)" % (ltarget, oe.package.is_elf(ltarget)))
990 symlinks[file] = target
991 continue 990 continue
991 # Use a reference of device ID and inode number to identify files
992 file_reference = "%d_%d" % (s.st_dev, s.st_ino)
993 checkelf[file] = (file, file_reference)
994
995 results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelflinks.values(), d)
996 results_map = {}
997 for (ltarget, elf_file) in results:
998 results_map[ltarget] = elf_file
999 for file in checkelflinks:
1000 ltarget = checkelflinks[file]
1001 # If it's a symlink, and points to an ELF file, we capture the readlink target
1002 if results_map[ltarget]:
1003 target = os.readlink(file)
1004 #bb.note("Sym: %s (%d)" % (ltarget, results_map[ltarget]))
1005 symlinks[file] = target
1006
1007 results = oe.utils.multiprocess_launch(oe.package.is_elf, checkelf.keys(), d)
1008 for (file, elf_file) in results:
1009 # It's a file (or hardlink), not a link
1010 # ...but is it ELF, and is it already stripped?
1011 if elf_file & 1:
1012 if elf_file & 2:
1013 if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split():
1014 bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn))
1015 else:
1016 msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn)
1017 package_qa_handle_error("already-stripped", msg, d)
1018 continue
992 1019
993 # It's a file (or hardlink), not a link 1020 # At this point we have an unstripped elf file. We need to:
994 # ...but is it ELF, and is it already stripped? 1021 # a) Make sure any file we strip is not hardlinked to anything else outside this tree
995 elf_file = oe.package.is_elf(file) 1022 # b) Only strip any hardlinked file once (no races)
996 if elf_file & 1: 1023 # c) Track any hardlinks between files so that we can reconstruct matching debug file hardlinks
997 if elf_file & 2: 1024
998 if 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split(): 1025 # Use a reference of device ID and inode number to identify files
999 bb.note("Skipping file %s from %s for already-stripped QA test" % (file[len(dvar):], pn)) 1026 file_reference = checkelf[file]
1000 else: 1027 if file_reference in inodes:
1001 msg = "File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dvar):], pn) 1028 os.unlink(file)
1002 package_qa_handle_error("already-stripped", msg, d) 1029 os.link(inodes[file_reference][0], file)
1003 continue 1030 inodes[file_reference].append(file)
1004 1031 else:
1005 # At this point we have an unstripped elf file. We need to: 1032 inodes[file_reference] = [file]
1006 # a) Make sure any file we strip is not hardlinked to anything else outside this tree 1033 # break hardlink
1007 # b) Only strip any hardlinked file once (no races) 1034 bb.utils.copyfile(file, file)
1008 # c) Track any hardlinks between files so that we can reconstruct matching debug file hardlinks 1035 elffiles[file] = elf_file
1009 1036 # Modified the file so clear the cache
1010 # Use a reference of device ID and inode number to identify files 1037 cpath.updatecache(file)
1011 file_reference = "%d_%d" % (s.st_dev, s.st_ino)
1012 if file_reference in inodes:
1013 os.unlink(file)
1014 os.link(inodes[file_reference][0], file)
1015 inodes[file_reference].append(file)
1016 else:
1017 inodes[file_reference] = [file]
1018 # break hardlink
1019 bb.utils.copyfile(file, file)
1020 elffiles[file] = elf_file
1021 # Modified the file so clear the cache
1022 cpath.updatecache(file)
1023 1038
1024 # 1039 #
1025 # First lets process debug splitting 1040 # First lets process debug splitting
diff --git a/meta/classes/staging.bbclass b/meta/classes/staging.bbclass
index 939042eb44..41df883495 100644
--- a/meta/classes/staging.bbclass
+++ b/meta/classes/staging.bbclass
@@ -70,7 +70,7 @@ sysroot_stage_all() {
70python sysroot_strip () { 70python sysroot_strip () {
71 inhibit_sysroot = d.getVar('INHIBIT_SYSROOT_STRIP') 71 inhibit_sysroot = d.getVar('INHIBIT_SYSROOT_STRIP')
72 if inhibit_sysroot and oe.types.boolean(inhibit_sysroot): 72 if inhibit_sysroot and oe.types.boolean(inhibit_sysroot):
73 return 0 73 return
74 74
75 dstdir = d.getVar('SYSROOT_DESTDIR') 75 dstdir = d.getVar('SYSROOT_DESTDIR')
76 pn = d.getVar('PN') 76 pn = d.getVar('PN')
@@ -79,7 +79,7 @@ python sysroot_strip () {
79 qa_already_stripped = 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split() 79 qa_already_stripped = 'already-stripped' in (d.getVar('INSANE_SKIP_' + pn) or "").split()
80 strip_cmd = d.getVar("STRIP") 80 strip_cmd = d.getVar("STRIP")
81 81
82 oe.package.strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, 82 oe.package.strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d,
83 qa_already_stripped=qa_already_stripped) 83 qa_already_stripped=qa_already_stripped)
84} 84}
85 85
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index a435d661a6..4255143371 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -74,7 +74,7 @@ def is_elf(path):
74 if "relocatable" in result: 74 if "relocatable" in result:
75 if path.endswith(".ko") and path.find("/lib/modules/") != -1 and is_kernel_module(path): 75 if path.endswith(".ko") and path.find("/lib/modules/") != -1 and is_kernel_module(path):
76 exec_type |= 16 76 exec_type |= 16
77 return exec_type 77 return (path, exec_type)
78 78
79def is_static_lib(path): 79def is_static_lib(path):
80 if path.endswith('.a') and not os.path.islink(path): 80 if path.endswith('.a') and not os.path.islink(path):
@@ -86,7 +86,7 @@ def is_static_lib(path):
86 return start == magic 86 return start == magic
87 return False 87 return False
88 88
89def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=False): 89def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, d, qa_already_stripped=False):
90 """ 90 """
91 Strip executable code (like executables, shared libraries) _in_place_ 91 Strip executable code (like executables, shared libraries) _in_place_
92 - Based on sysroot_strip in staging.bbclass 92 - Based on sysroot_strip in staging.bbclass
@@ -107,6 +107,8 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
107 # 107 #
108 # First lets figure out all of the files we may have to process 108 # First lets figure out all of the files we may have to process
109 # 109 #
110 checkelf = []
111 inodecache = {}
110 for root, dirs, files in os.walk(dstdir): 112 for root, dirs, files in os.walk(dstdir):
111 for f in files: 113 for f in files:
112 file = os.path.join(root, f) 114 file = os.path.join(root, f)
@@ -132,7 +134,11 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
132 134
133 # It's a file (or hardlink), not a link 135 # It's a file (or hardlink), not a link
134 # ...but is it ELF, and is it already stripped? 136 # ...but is it ELF, and is it already stripped?
135 elf_file = is_elf(file) 137 checkelf.append(file)
138 inodecache[file] = s.st_ino
139 results = oe.utils.multiprocess_launch(is_elf, checkelf, d)
140 for (file, elf_file) in results:
141 #elf_file = is_elf(file)
136 if elf_file & 1: 142 if elf_file & 1:
137 if elf_file & 2: 143 if elf_file & 2:
138 if qa_already_stripped: 144 if qa_already_stripped:
@@ -141,12 +147,12 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir, qa_already_stripped=
141 bb.warn("File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dstdir):], pn)) 147 bb.warn("File '%s' from %s was already stripped, this will prevent future debugging!" % (file[len(dstdir):], pn))
142 continue 148 continue
143 149
144 if s.st_ino in inodes: 150 if inodecache[file] in inodes:
145 os.unlink(file) 151 os.unlink(file)
146 os.link(inodes[s.st_ino], file) 152 os.link(inodes[inodecache[file]], file)
147 else: 153 else:
148 # break hardlinks so that we do not strip the original. 154 # break hardlinks so that we do not strip the original.
149 inodes[s.st_ino] = file 155 inodes[inodecache[file]] = file
150 bb.utils.copyfile(file, file) 156 bb.utils.copyfile(file, file)
151 elffiles[file] = elf_file 157 elffiles[file] = elf_file
152 158