summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2013-03-14 17:26:20 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2013-03-18 21:26:40 +0000
commitadd11fa1abfc51f1a793c84f02cd7d85d793fe14 (patch)
treee58176a867d272a57d91f61e2f954fc0249d007e
parentcec0102647e3f2c93de5125c1de2436b4b787bdd (diff)
downloadpoky-add11fa1abfc51f1a793c84f02cd7d85d793fe14.tar.gz
package: Add cachedpath optimisation
Currently, various standard library operations like os.walk(), os.path.isdir() and os.path.islink() each call stat or lstat which involves a syscall into the kernel. There is no caching since they could conceivably have changed on disk. The result is that for something like the do_package task of the kernel we're spending over two minutes making 868,000 individual stat calls for 23,000 files. This is suboptimal. This patch adds lib/oe/cachedpath.py which are a set of replacement functions for these operations which use cached stat data rather than hitting the kernel each time. It gives a nice performance improvement halving the build time of the kernel do_package. (From OE-Core rev: 556dee0c4d6d8a87c0cddbd2f60fe5917d009f18) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/base.bbclass2
-rw-r--r--meta/classes/package.bbclass56
-rw-r--r--meta/lib/oe/cachedpath.py235
3 files changed, 268 insertions, 25 deletions
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 4ec1eda6bc..5fe9a84efe 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -10,7 +10,7 @@ inherit utility-tasks
10inherit metadata_scm 10inherit metadata_scm
11inherit logging 11inherit logging
12 12
13OE_IMPORTS += "os sys time oe.path oe.utils oe.data oe.package oe.packagegroup oe.sstatesig oe.lsb" 13OE_IMPORTS += "os sys time oe.path oe.utils oe.data oe.package oe.packagegroup oe.sstatesig oe.lsb oe.cachedpath"
14OE_IMPORTS[type] = "list" 14OE_IMPORTS[type] = "list"
15 15
16def oe_import(d): 16def oe_import(d):
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index f3a6bc726d..b6f87674a0 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -288,7 +288,7 @@ def copydebugsources(debugsrcdir, d):
288 basepath = dvar 288 basepath = dvar
289 for p in debugsrcdir.split("/"): 289 for p in debugsrcdir.split("/"):
290 basepath = basepath + "/" + p 290 basepath = basepath + "/" + p
291 if not os.path.exists(basepath): 291 if not cpath.exists(basepath):
292 nosuchdir.append(basepath) 292 nosuchdir.append(basepath)
293 bb.utils.mkdirhier(basepath) 293 bb.utils.mkdirhier(basepath)
294 294
@@ -388,7 +388,7 @@ python package_do_split_locales() {
388 388
389 localedir = os.path.join(dvar + datadir, 'locale') 389 localedir = os.path.join(dvar + datadir, 'locale')
390 390
391 if not os.path.isdir(localedir): 391 if not cpath.isdir(localedir):
392 bb.debug(1, "No locale files in this package") 392 bb.debug(1, "No locale files in this package")
393 return 393 return
394 394
@@ -628,7 +628,7 @@ python fixup_perms () {
628 continue 628 continue
629 629
630 origin = dvar + dir 630 origin = dvar + dir
631 if not (os.path.exists(origin) and os.path.isdir(origin) and not os.path.islink(origin)): 631 if not (cpath.exists(origin) and cpath.isdir(origin) and not cpath.islink(origin)):
632 continue 632 continue
633 633
634 link = fs_perms_table[dir].link 634 link = fs_perms_table[dir].link
@@ -654,7 +654,7 @@ python fixup_perms () {
654 continue 654 continue
655 655
656 origin = dvar + dir 656 origin = dvar + dir
657 if not (os.path.exists(origin) and os.path.isdir(origin)): 657 if not (cpath.exists(origin) and cpath.isdir(origin)):
658 continue 658 continue
659 659
660 fix_perms(origin, fs_perms_table[dir].mode, fs_perms_table[dir].uid, fs_perms_table[dir].gid, dir) 660 fix_perms(origin, fs_perms_table[dir].mode, fs_perms_table[dir].uid, fs_perms_table[dir].gid, dir)
@@ -735,7 +735,7 @@ python split_and_strip_files () {
735 baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir", True)) 735 baselibdir = os.path.abspath(dvar + os.sep + d.getVar("base_libdir", True))
736 if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT', True) != '1') and \ 736 if (d.getVar('INHIBIT_PACKAGE_DEBUG_SPLIT', True) != '1') and \
737 (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'): 737 (d.getVar('INHIBIT_PACKAGE_STRIP', True) != '1'):
738 for root, dirs, files in os.walk(dvar): 738 for root, dirs, files in cpath.walk(dvar):
739 for f in files: 739 for f in files:
740 file = os.path.join(root, f) 740 file = os.path.join(root, f)
741 if file.endswith(".ko") and file.find("/lib/modules/") != -1: 741 if file.endswith(".ko") and file.find("/lib/modules/") != -1:
@@ -749,18 +749,20 @@ python split_and_strip_files () {
749 continue 749 continue
750 750
751 try: 751 try:
752 ltarget = oe.path.realpath(file, dvar, False) 752 ltarget = cpath.realpath(file, dvar, False)
753 s = os.lstat(ltarget) 753 s = cpath.lstat(ltarget)
754 except OSError, (err, strerror): 754 except OSError, (err, strerror):
755 if err != errno.ENOENT: 755 if err != errno.ENOENT:
756 raise 756 raise
757 # Skip broken symlinks 757 # Skip broken symlinks
758 continue 758 continue
759 if not s:
760 continue
759 # Check its an excutable 761 # Check its an excutable
760 if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \ 762 if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] & stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
761 or ((file.startswith(libdir) or file.startswith(baselibdir)) and ".so" in f): 763 or ((file.startswith(libdir) or file.startswith(baselibdir)) and ".so" in f):
762 # If it's a symlink, and points to an ELF file, we capture the readlink target 764 # If it's a symlink, and points to an ELF file, we capture the readlink target
763 if os.path.islink(file): 765 if cpath.islink(file):
764 target = os.readlink(file) 766 target = os.readlink(file)
765 if isELF(ltarget): 767 if isELF(ltarget):
766 #bb.note("Sym: %s (%d)" % (ltarget, isELF(ltarget))) 768 #bb.note("Sym: %s (%d)" % (ltarget, isELF(ltarget)))
@@ -918,8 +920,8 @@ python populate_packages () {
918 for file in files: 920 for file in files:
919 if os.path.isabs(file): 921 if os.path.isabs(file):
920 file = '.' + file 922 file = '.' + file
921 if not os.path.islink(file): 923 if not cpath.islink(file):
922 if os.path.isdir(file): 924 if cpath.isdir(file):
923 newfiles = [ os.path.join(file,x) for x in os.listdir(file) ] 925 newfiles = [ os.path.join(file,x) for x in os.listdir(file) ]
924 if newfiles: 926 if newfiles:
925 files += newfiles 927 files += newfiles
@@ -929,7 +931,7 @@ python populate_packages () {
929 if [ file ] != globbed: 931 if [ file ] != globbed:
930 files += globbed 932 files += globbed
931 continue 933 continue
932 if (not os.path.islink(file)) and (not os.path.exists(file)): 934 if (not cpath.islink(file)) and (not cpath.exists(file)):
933 continue 935 continue
934 if file in seen: 936 if file in seen:
935 continue 937 continue
@@ -938,33 +940,33 @@ python populate_packages () {
938 def mkdir(src, dest, p): 940 def mkdir(src, dest, p):
939 src = os.path.join(src, p) 941 src = os.path.join(src, p)
940 dest = os.path.join(dest, p) 942 dest = os.path.join(dest, p)
941 bb.utils.mkdirhier(dest) 943 fstat = cpath.stat(src)
942 fstat = os.stat(src) 944 os.mkdir(dest, fstat.st_mode)
943 os.chmod(dest, fstat.st_mode)
944 os.chown(dest, fstat.st_uid, fstat.st_gid) 945 os.chown(dest, fstat.st_uid, fstat.st_gid)
945 if p not in seen: 946 if p not in seen:
946 seen.append(p) 947 seen.append(p)
948 cpath.updatecache(dest)
947 949
948 def mkdir_recurse(src, dest, paths): 950 def mkdir_recurse(src, dest, paths):
949 if os.path.exists(dest + '/' + paths): 951 if cpath.exists(dest + '/' + paths):
950 return 952 return
951 while paths.startswith("./"): 953 while paths.startswith("./"):
952 paths = paths[2:] 954 paths = paths[2:]
953 p = "." 955 p = "."
954 for c in paths.split("/"): 956 for c in paths.split("/"):
955 p = os.path.join(p, c) 957 p = os.path.join(p, c)
956 if not os.path.exists(os.path.join(dest, p)): 958 if not cpath.exists(os.path.join(dest, p)):
957 mkdir(src, dest, p) 959 mkdir(src, dest, p)
958 960
959 if os.path.isdir(file) and not os.path.islink(file): 961 if cpath.isdir(file) and not cpath.islink(file):
960 mkdir_recurse(dvar, root, file) 962 mkdir_recurse(dvar, root, file)
961 continue 963 continue
962 964
963 mkdir_recurse(dvar, root, os.path.dirname(file)) 965 mkdir_recurse(dvar, root, os.path.dirname(file))
964 fpath = os.path.join(root,file) 966 fpath = os.path.join(root,file)
965 if not os.path.islink(file): 967 if not cpath.islink(file):
966 os.link(file, fpath) 968 os.link(file, fpath)
967 fstat = os.stat(file) 969 fstat = cpath.stat(file)
968 os.chmod(fpath, fstat.st_mode) 970 os.chmod(fpath, fstat.st_mode)
969 os.chown(fpath, fstat.st_uid, fstat.st_gid) 971 os.chown(fpath, fstat.st_uid, fstat.st_gid)
970 continue 972 continue
@@ -975,7 +977,7 @@ python populate_packages () {
975 os.chdir(workdir) 977 os.chdir(workdir)
976 978
977 unshipped = [] 979 unshipped = []
978 for root, dirs, files in os.walk(dvar): 980 for root, dirs, files in cpath.walk(dvar):
979 dir = root[len(dvar):] 981 dir = root[len(dvar):]
980 if not dir: 982 if not dir:
981 dir = os.sep 983 dir = os.sep
@@ -1009,8 +1011,8 @@ python package_fixsymlinks () {
1009 for path in pkgfiles[pkg]: 1011 for path in pkgfiles[pkg]:
1010 rpath = path[len(inst_root):] 1012 rpath = path[len(inst_root):]
1011 pkg_files[pkg].append(rpath) 1013 pkg_files[pkg].append(rpath)
1012 rtarget = oe.path.realpath(path, inst_root, True, assume_dir = True) 1014 rtarget = cpath.realpath(path, inst_root, True, assume_dir = True)
1013 if not os.path.lexists(rtarget): 1015 if not cpath.lexists(rtarget):
1014 dangling_links[pkg].append(os.path.normpath(rtarget[len(inst_root):])) 1016 dangling_links[pkg].append(os.path.normpath(rtarget[len(inst_root):]))
1015 1017
1016 newrdepends = {} 1018 newrdepends = {}
@@ -1394,7 +1396,7 @@ python package_do_shlibs() {
1394 renames = list() 1396 renames = list()
1395 for file in pkgfiles[pkg]: 1397 for file in pkgfiles[pkg]:
1396 soname = None 1398 soname = None
1397 if os.path.islink(file): 1399 if cpath.islink(file):
1398 continue 1400 continue
1399 if targetos == "darwin" or targetos == "darwin8": 1401 if targetos == "darwin" or targetos == "darwin8":
1400 darwin_so(file) 1402 darwin_so(file)
@@ -1781,6 +1783,10 @@ python do_package () {
1781 # as any change to rpmdeps requires this to be rerun. 1783 # as any change to rpmdeps requires this to be rerun.
1782 # PACKAGE_BBCLASS_VERSION = "1" 1784 # PACKAGE_BBCLASS_VERSION = "1"
1783 1785
1786 # Init cachedpath
1787 global cpath
1788 cpath = oe.cachedpath.CachedPath()
1789
1784 ########################################################################### 1790 ###########################################################################
1785 # Sanity test the setup 1791 # Sanity test the setup
1786 ########################################################################### 1792 ###########################################################################
@@ -1827,6 +1833,8 @@ python do_package () {
1827 # Split up PKGD into PKGDEST 1833 # Split up PKGD into PKGDEST
1828 ########################################################################### 1834 ###########################################################################
1829 1835
1836 cpath = oe.cachedpath.CachedPath()
1837
1830 for f in (d.getVar('PACKAGESPLITFUNCS', True) or '').split(): 1838 for f in (d.getVar('PACKAGESPLITFUNCS', True) or '').split():
1831 bb.build.exec_func(f, d) 1839 bb.build.exec_func(f, d)
1832 1840
@@ -1841,7 +1849,7 @@ python do_package () {
1841 pkgdest = d.getVar('PKGDEST', True) 1849 pkgdest = d.getVar('PKGDEST', True)
1842 for pkg in packages: 1850 for pkg in packages:
1843 pkgfiles[pkg] = [] 1851 pkgfiles[pkg] = []
1844 for walkroot, dirs, files in os.walk(pkgdest + "/" + pkg): 1852 for walkroot, dirs, files in cpath.walk(pkgdest + "/" + pkg):
1845 for file in files: 1853 for file in files:
1846 pkgfiles[pkg].append(walkroot + os.sep + file) 1854 pkgfiles[pkg].append(walkroot + os.sep + file)
1847 1855
diff --git a/meta/lib/oe/cachedpath.py b/meta/lib/oe/cachedpath.py
new file mode 100644
index 0000000000..e350c8a70e
--- /dev/null
+++ b/meta/lib/oe/cachedpath.py
@@ -0,0 +1,235 @@
1#
2# Based on standard python library functions but avoid
3# repeated stat calls. Its assumed the files will not change from under us
4# so we can cache stat calls.
5#
6
7import os
8import errno
9import stat as statmod
10
11class CachedPath(object):
12 def __init__(self):
13 self.statcache = {}
14 self.lstatcache = {}
15 self.normpathcache = {}
16 return
17
18 def updatecache(self, x):
19 x = self.normpath(x)
20 if x in self.statcache:
21 del self.statcache[x]
22 if x in self.lstatcache:
23 del self.lstatcache[x]
24
25 def normpath(self, path):
26 if path in self.normpathcache:
27 return self.normpathcache[path]
28 newpath = os.path.normpath(path)
29 self.normpathcache[path] = newpath
30 return newpath
31
32 def _callstat(self, path):
33 if path in self.statcache:
34 return self.statcache[path]
35 try:
36 st = os.stat(path)
37 self.statcache[path] = st
38 return st
39 except os.error:
40 self.statcache[path] = False
41 return False
42
43 # We might as well call lstat and then only
44 # call stat as well in the symbolic link case
45 # since this turns out to be much more optimal
46 # in real world usage of this cache
47 def callstat(self, path):
48 path = self.normpath(path)
49 self.calllstat(path)
50 return self.statcache[path]
51
52 def calllstat(self, path):
53 path = self.normpath(path)
54 if path in self.lstatcache:
55 return self.lstatcache[path]
56 #bb.error("LStatpath:" + path)
57 try:
58 lst = os.lstat(path)
59 self.lstatcache[path] = lst
60 if not statmod.S_ISLNK(lst.st_mode):
61 self.statcache[path] = lst
62 else:
63 self._callstat(path)
64 return lst
65 except (os.error, AttributeError):
66 self.lstatcache[path] = False
67 self.statcache[path] = False
68 return False
69
70 # This follows symbolic links, so both islink() and isdir() can be true
71 # for the same path ono systems that support symlinks
72 def isfile(self, path):
73 """Test whether a path is a regular file"""
74 st = self.callstat(path)
75 if not st:
76 return False
77 return statmod.S_ISREG(st.st_mode)
78
79 # Is a path a directory?
80 # This follows symbolic links, so both islink() and isdir()
81 # can be true for the same path on systems that support symlinks
82 def isdir(self, s):
83 """Return true if the pathname refers to an existing directory."""
84 st = self.callstat(s)
85 if not st:
86 return False
87 return statmod.S_ISDIR(st.st_mode)
88
89 def islink(self, path):
90 """Test whether a path is a symbolic link"""
91 st = self.calllstat(path)
92 if not st:
93 return False
94 return statmod.S_ISLNK(st.st_mode)
95
96 # Does a path exist?
97 # This is false for dangling symbolic links on systems that support them.
98 def exists(self, path):
99 """Test whether a path exists. Returns False for broken symbolic links"""
100 if self.callstat(path):
101 return True
102 return False
103
104 def lexists(self, path):
105 """Test whether a path exists. Returns True for broken symbolic links"""
106 if self.calllstat(path):
107 return True
108 return False
109
110 def stat(self, path):
111 return self.callstat(path)
112
113 def lstat(self, path):
114 return self.calllstat(path)
115
116 def walk(self, top, topdown=True, onerror=None, followlinks=False):
117 # Matches os.walk, not os.path.walk()
118
119 # We may not have read permission for top, in which case we can't
120 # get a list of the files the directory contains. os.path.walk
121 # always suppressed the exception then, rather than blow up for a
122 # minor reason when (say) a thousand readable directories are still
123 # left to visit. That logic is copied here.
124 try:
125 # Note that listdir and error are globals in this module due
126 # to earlier import-*.
127 names = os.listdir(top)
128 except error, err:
129 if onerror is not None:
130 onerror(err)
131 return
132
133 dirs, nondirs = [], []
134 for name in names:
135 if self.isdir(os.path.join(top, name)):
136 dirs.append(name)
137 else:
138 nondirs.append(name)
139
140 if topdown:
141 yield top, dirs, nondirs
142 for name in dirs:
143 new_path = os.path.join(top, name)
144 if followlinks or not self.islink(new_path):
145 for x in self.walk(new_path, topdown, onerror, followlinks):
146 yield x
147 if not topdown:
148 yield top, dirs, nondirs
149
150 ## realpath() related functions
151 def __is_path_below(self, file, root):
152 return (file + os.path.sep).startswith(root)
153
154 def __realpath_rel(self, start, rel_path, root, loop_cnt, assume_dir):
155 """Calculates real path of symlink 'start' + 'rel_path' below
156 'root'; no part of 'start' below 'root' must contain symlinks. """
157 have_dir = True
158
159 for d in rel_path.split(os.path.sep):
160 if not have_dir and not assume_dir:
161 raise OSError(errno.ENOENT, "no such directory %s" % start)
162
163 if d == os.path.pardir: # '..'
164 if len(start) >= len(root):
165 # do not follow '..' before root
166 start = os.path.dirname(start)
167 else:
168 # emit warning?
169 pass
170 else:
171 (start, have_dir) = self.__realpath(os.path.join(start, d),
172 root, loop_cnt, assume_dir)
173
174 assert(self.__is_path_below(start, root))
175
176 return start
177
178 def __realpath(self, file, root, loop_cnt, assume_dir):
179 while self.islink(file) and len(file) >= len(root):
180 if loop_cnt == 0:
181 raise OSError(errno.ELOOP, file)
182
183 loop_cnt -= 1
184 target = os.path.normpath(os.readlink(file))
185
186 if not os.path.isabs(target):
187 tdir = os.path.dirname(file)
188 assert(self.__is_path_below(tdir, root))
189 else:
190 tdir = root
191
192 file = self.__realpath_rel(tdir, target, root, loop_cnt, assume_dir)
193
194 try:
195 is_dir = self.isdir(file)
196 except:
197 is_dir = False
198
199 return (file, is_dir)
200
201 def realpath(self, file, root, use_physdir = True, loop_cnt = 100, assume_dir = False):
202 """ Returns the canonical path of 'file' with assuming a
203 toplevel 'root' directory. When 'use_physdir' is set, all
204 preceding path components of 'file' will be resolved first;
205 this flag should be set unless it is guaranteed that there is
206 no symlink in the path. When 'assume_dir' is not set, missing
207 path components will raise an ENOENT error"""
208
209 root = os.path.normpath(root)
210 file = os.path.normpath(file)
211
212 if not root.endswith(os.path.sep):
213 # letting root end with '/' makes some things easier
214 root = root + os.path.sep
215
216 if not self.__is_path_below(file, root):
217 raise OSError(errno.EINVAL, "file '%s' is not below root" % file)
218
219 try:
220 if use_physdir:
221 file = self.__realpath_rel(root, file[(len(root) - 1):], root, loop_cnt, assume_dir)
222 else:
223 file = self.__realpath(file, root, loop_cnt, assume_dir)[0]
224 except OSError, e:
225 if e.errno == errno.ELOOP:
226 # make ELOOP more readable; without catching it, there will
227 # be printed a backtrace with 100s of OSError exceptions
228 # else
229 raise OSError(errno.ELOOP,
230 "too much recursions while resolving '%s'; loop in '%s'" %
231 (file, e.strerror))
232
233 raise
234
235 return file