summaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2013-02-01 13:50:38 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2013-02-06 13:12:59 +0000
commitd81aa06ece4e5dcae5f91ab0ae8958d58cd3419f (patch)
tree9d8bdc7fa6f5cadcc4143f3ed48416351ed13790 /meta
parent6c6f6b7ffd770418bda92898a5fdf90877e74dde (diff)
downloadpoky-d81aa06ece4e5dcae5f91ab0ae8958d58cd3419f.tar.gz
package.bbclass: Multithread per file dependency generation code
(From OE-Core rev: b659eb0f2070149d9516c129b3853b41fbbd1033) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
-rw-r--r--meta/classes/base.bbclass2
-rw-r--r--meta/classes/package.bbclass92
-rw-r--r--meta/classes/update-alternatives.bbclass2
-rw-r--r--meta/lib/oe/package.py51
4 files changed, 85 insertions, 62 deletions
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 4662d3bf64..5f43733905 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -10,7 +10,7 @@ inherit utility-tasks
10inherit metadata_scm 10inherit metadata_scm
11inherit logging 11inherit logging
12 12
13OE_IMPORTS += "os sys time oe.path oe.utils oe.data oe.packagegroup oe.sstatesig oe.lsb" 13OE_IMPORTS += "os sys time oe.path oe.utils oe.data oe.package oe.packagegroup oe.sstatesig oe.lsb"
14OE_IMPORTS[type] = "list" 14OE_IMPORTS[type] = "list"
15 15
16def oe_import(d): 16def oe_import(d):
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 2fbd2602ca..1858836697 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -1236,85 +1236,57 @@ RPMDEPS = "${STAGING_LIBDIR_NATIVE}/rpm/bin/rpmdeps-oecore --macros ${STAGING_LI
1236# FILERDEPENDS_filepath_pkg - per file dep 1236# FILERDEPENDS_filepath_pkg - per file dep
1237 1237
1238python package_do_filedeps() { 1238python package_do_filedeps() {
1239 import re
1240
1241 if d.getVar('SKIP_FILEDEPS', True) == '1': 1239 if d.getVar('SKIP_FILEDEPS', True) == '1':
1242 return 1240 return
1243 1241
1244 pkgdest = d.getVar('PKGDEST', True) 1242 pkgdest = d.getVar('PKGDEST', True)
1245 packages = d.getVar('PACKAGES', True) 1243 packages = d.getVar('PACKAGES', True)
1246
1247 rpmdeps = d.expand("${RPMDEPS}") 1244 rpmdeps = d.expand("${RPMDEPS}")
1248 r = re.compile(r'[<>=]+ +[^ ]*')
1249
1250 def file_translate(file):
1251 ft = file.replace("@", "@at@")
1252 ft = ft.replace(" ", "@space@")
1253 ft = ft.replace("\t", "@tab@")
1254 ft = ft.replace("[", "@openbrace@")
1255 ft = ft.replace("]", "@closebrace@")
1256 ft = ft.replace("_", "@underscore@")
1257 return ft
1258
1259 # Quick routine to process the results of the rpmdeps call...
1260 def process_deps(pipe, pkg, provides_files, requires_files):
1261 provides = {}
1262 requires = {}
1263
1264 for line in pipe:
1265 f = line.split(" ", 1)[0].strip()
1266 line = line.split(" ", 1)[1].strip()
1267
1268 if line.startswith("Requires:"):
1269 i = requires
1270 elif line.startswith("Provides:"):
1271 i = provides
1272 else:
1273 continue
1274
1275 file = f.replace(pkgdest + "/" + pkg, "")
1276 file = file_translate(file)
1277 value = line.split(":", 1)[1].strip()
1278 value = r.sub(r'(\g<0>)', value)
1279
1280 if value.startswith("rpmlib("):
1281 continue
1282 if value == "python":
1283 continue
1284 if file not in i:
1285 i[file] = []
1286 i[file].append(value)
1287
1288 for file in provides:
1289 provides_files.append(file)
1290 key = "FILERPROVIDES_" + file + "_" + pkg
1291 d.setVar(key, " ".join(provides[file]))
1292
1293 for file in requires:
1294 requires_files.append(file)
1295 key = "FILERDEPENDS_" + file + "_" + pkg
1296 d.setVar(key, " ".join(requires[file]))
1297 1245
1298 def chunks(files, n): 1246 def chunks(files, n):
1299 return [files[i:i+n] for i in range(0, len(files), n)] 1247 return [files[i:i+n] for i in range(0, len(files), n)]
1300 1248
1301 # Determine dependencies 1249 pkglist = []
1302 for pkg in packages.split(): 1250 for pkg in packages.split():
1303 if d.getVar('SKIP_FILEDEPS_' + pkg, True) == '1': 1251 if d.getVar('SKIP_FILEDEPS_' + pkg, True) == '1':
1304 continue 1252 continue
1305 if pkg.endswith('-dbg') or pkg.endswith('-doc') or pkg.find('-locale-') != -1 or pkg.find('-localedata-') != -1 or pkg.find('-gconv-') != -1 or pkg.find('-charmap-') != -1 or pkg.startswith('kernel-module-'): 1253 if pkg.endswith('-dbg') or pkg.endswith('-doc') or pkg.find('-locale-') != -1 or pkg.find('-localedata-') != -1 or pkg.find('-gconv-') != -1 or pkg.find('-charmap-') != -1 or pkg.startswith('kernel-module-'):
1306 continue 1254 continue
1255 for files in chunks(pkgfiles[pkg], 100):
1256 pkglist.append((pkg, files, rpmdeps, pkgdest))
1307 1257
1308 provides_files = [] 1258 import multiprocessing
1309 requires_files = [] 1259 nproc = multiprocessing.cpu_count()
1260 pool = multiprocessing.Pool(nproc)
1261 processed = pool.imap(oe.package.filedeprunner, pkglist)
1262 pool.close()
1263 pool.join()
1310 1264
1311 for files in chunks(pkgfiles[pkg], 100): 1265 provides_files = {}
1312 dep_pipe = os.popen(rpmdeps + " " + " ".join(files)) 1266 requires_files = {}
1267
1268 for result in processed:
1269 (pkg, provides, requires) = result
1270
1271 if pkg not in provides_files:
1272 provides_files[pkg] = []
1273 if pkg not in requires_files:
1274 requires_files[pkg] = []
1313 1275
1314 process_deps(dep_pipe, pkg, provides_files, requires_files) 1276 for file in provides:
1277 provides_files[pkg].append(file)
1278 key = "FILERPROVIDES_" + file + "_" + pkg
1279 d.setVar(key, " ".join(provides[file]))
1280
1281 for file in requires:
1282 requires_files[pkg].append(file)
1283 key = "FILERDEPENDS_" + file + "_" + pkg
1284 d.setVar(key, " ".join(requires[file]))
1315 1285
1316 d.setVar("FILERDEPENDSFLIST_" + pkg, " ".join(requires_files)) 1286 for pkg in requires_files:
1317 d.setVar("FILERPROVIDESFLIST_" + pkg, " ".join(provides_files)) 1287 d.setVar("FILERDEPENDSFLIST_" + pkg, " ".join(requires_files[pkg]))
1288 for pkg in provides_files:
1289 d.setVar("FILERPROVIDESFLIST_" + pkg, " ".join(provides_files[pkg]))
1318} 1290}
1319 1291
1320def getshlibsdirs(d): 1292def getshlibsdirs(d):
diff --git a/meta/classes/update-alternatives.bbclass b/meta/classes/update-alternatives.bbclass
index a75e282a3c..90bc56b9fb 100644
--- a/meta/classes/update-alternatives.bbclass
+++ b/meta/classes/update-alternatives.bbclass
@@ -353,7 +353,7 @@ python package_do_filedeps_append () {
353 continue 353 continue
354 354
355 # Add file provide 355 # Add file provide
356 trans_target = file_translate(alt_target) 356 trans_target = oe.package.file_translate(alt_target)
357 d.appendVar('FILERPROVIDES_%s_%s' % (trans_target, pkg), " " + alt_link) 357 d.appendVar('FILERPROVIDES_%s_%s' % (trans_target, pkg), " " + alt_link)
358 if not trans_target in (d.getVar('FILERPROVIDESFLIST_%s' % pkg, True) or ""): 358 if not trans_target in (d.getVar('FILERPROVIDESFLIST_%s' % pkg, True) or ""):
359 d.appendVar('FILERPROVIDESFLIST_%s' % pkg, " " + trans_target) 359 d.appendVar('FILERPROVIDESFLIST_%s' % pkg, " " + trans_target)
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
new file mode 100644
index 0000000000..6b1c1f48ce
--- /dev/null
+++ b/meta/lib/oe/package.py
@@ -0,0 +1,51 @@
1
2def file_translate(file):
3 ft = file.replace("@", "@at@")
4 ft = ft.replace(" ", "@space@")
5 ft = ft.replace("\t", "@tab@")
6 ft = ft.replace("[", "@openbrace@")
7 ft = ft.replace("]", "@closebrace@")
8 ft = ft.replace("_", "@underscore@")
9 return ft
10
11def filedeprunner(arg):
12 import re
13
14 (pkg, pkgfiles, rpmdeps, pkgdest) = arg
15 provides = {}
16 requires = {}
17
18 r = re.compile(r'[<>=]+ +[^ ]*')
19
20 def process_deps(pipe, pkg, pkgdest, provides, requires):
21 for line in pipe:
22 f = line.split(" ", 1)[0].strip()
23 line = line.split(" ", 1)[1].strip()
24
25 if line.startswith("Requires:"):
26 i = requires
27 elif line.startswith("Provides:"):
28 i = provides
29 else:
30 continue
31
32 file = f.replace(pkgdest + "/" + pkg, "")
33 file = file_translate(file)
34 value = line.split(":", 1)[1].strip()
35 value = r.sub(r'(\g<0>)', value)
36
37 if value.startswith("rpmlib("):
38 continue
39 if value == "python":
40 continue
41 if file not in i:
42 i[file] = []
43 i[file].append(value)
44
45 return provides, requires
46
47 dep_pipe = os.popen(rpmdeps + " " + " ".join(pkgfiles))
48
49 provides, requires = process_deps(dep_pipe, pkg, pkgdest, provides, requires)
50
51 return (pkg, provides, requires)