summaryrefslogtreecommitdiffstats
path: root/meta/lib/oe/package_manager.py
diff options
context:
space:
mode:
authorLaurentiu Palcu <laurentiu.palcu@intel.com>2014-02-14 13:01:05 +0200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-02-14 12:30:19 +0000
commita0a045098115d94d31f6fcd3c16dd286ce16d84a (patch)
tree93649f13ee5a5a6d0bcb1d59f90b482e81c254cd /meta/lib/oe/package_manager.py
parentaed96f6a6aee74d677c4647af159174477c134ab (diff)
downloadpoky-a0a045098115d94d31f6fcd3c16dd286ce16d84a.tar.gz
package_manager.py, rootfs.py, sdk.py: add Indexer class
Because the package-index.bb needs to create package indexes outside do_rootfs environment, move the indexing capability out of PackageManager class to a smaller Indexer class. This commit: * simply moves the indexing functions for ipk/deb with no changes; * rewrites the RPM indexing function so that it can be easily moved out of the PackageManager class; * removes some RPM duplicate code, moves it into a method inside RpmPM class and changes the RpmPM constructor so that the new method is effective; (From OE-Core rev: d339d6f48f81330e94162f333aad76f3c65d6bfd) Signed-off-by: Laurentiu Palcu <laurentiu.palcu@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib/oe/package_manager.py')
-rw-r--r--meta/lib/oe/package_manager.py339
1 files changed, 216 insertions, 123 deletions
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 9884c3a712..af14d5ad7f 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -22,6 +22,144 @@ def create_index(arg):
22 return None 22 return None
23 23
24 24
25class Indexer(object):
26 __metaclass__ = ABCMeta
27
28 def __init__(self, d, deploy_dir):
29 self.d = d
30 self.deploy_dir = deploy_dir
31
32 @abstractmethod
33 def write_index(self):
34 pass
35
36
37class RpmIndexer(Indexer):
38 def write_index(self):
39 sdk_pkg_archs = (self.d.getVar('SDK_PACKAGE_ARCHS', True) or "").replace('-', '_').split()
40 mlb_prefix_list = (self.d.getVar('MULTILIB_PREFIX_LIST', True) or "").replace('-', '_').split()
41 all_mlb_pkg_archs = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").replace('-', '_').split()
42
43 archs = set()
44 for item in mlb_prefix_list:
45 archs = archs.union(set(item.split(':')[1:]))
46
47 if len(archs) == 0:
48 archs = archs.union(set(all_mlb_pkg_archs))
49
50 archs = archs.union(set(sdk_pkg_archs))
51
52 rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo")
53 index_cmds = []
54 rpm_dirs_found = False
55 for arch in archs:
56 arch_dir = os.path.join(self.deploy_dir, arch)
57 if not os.path.isdir(arch_dir):
58 continue
59
60 index_cmds.append("%s --update -q %s" % (rpm_createrepo, arch_dir))
61
62 rpm_dirs_found = True
63
64 if not rpm_dirs_found:
65 return("There are no packages in %s" % self.deploy_dir)
66
67 nproc = multiprocessing.cpu_count()
68 pool = bb.utils.multiprocessingpool(nproc)
69 results = list(pool.imap(create_index, index_cmds))
70 pool.close()
71 pool.join()
72
73 for result in results:
74 if result is not None:
75 return(result)
76
77
78class OpkgIndexer(Indexer):
79 def write_index(self):
80 arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
81 "SDK_PACKAGE_ARCHS",
82 "MULTILIB_ARCHS"]
83
84 opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index")
85
86 if not os.path.exists(os.path.join(self.deploy_dir, "Packages")):
87 open(os.path.join(self.deploy_dir, "Packages"), "w").close()
88
89 index_cmds = []
90 for arch_var in arch_vars:
91 archs = self.d.getVar(arch_var, True)
92 if archs is None:
93 continue
94
95 for arch in archs.split():
96 pkgs_dir = os.path.join(self.deploy_dir, arch)
97 pkgs_file = os.path.join(pkgs_dir, "Packages")
98
99 if not os.path.isdir(pkgs_dir):
100 continue
101
102 if not os.path.exists(pkgs_file):
103 open(pkgs_file, "w").close()
104
105 index_cmds.append('%s -r %s -p %s -m %s' %
106 (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir))
107
108 if len(index_cmds) == 0:
109 return("There are no packages in %s!" % self.deploy_dir)
110
111 nproc = multiprocessing.cpu_count()
112 pool = bb.utils.multiprocessingpool(nproc)
113 results = list(pool.imap(create_index, index_cmds))
114 pool.close()
115 pool.join()
116
117 for result in results:
118 if result is not None:
119 return(result)
120
121
122class DpkgIndexer(Indexer):
123 def write_index(self):
124 pkg_archs = self.d.getVar('PACKAGE_ARCHS', True)
125 if pkg_archs is not None:
126 arch_list = pkg_archs.split()
127 sdk_pkg_archs = self.d.getVar('SDK_PACKAGE_ARCHS', True)
128 if sdk_pkg_archs is not None:
129 arch_list += sdk_pkg_archs.split()
130
131 dpkg_scanpackages = bb.utils.which(os.getenv('PATH'), "dpkg-scanpackages")
132 gzip = bb.utils.which(os.getenv('PATH'), "gzip")
133
134 index_cmds = []
135 deb_dirs_found = False
136 for arch in arch_list:
137 arch_dir = os.path.join(self.deploy_dir, arch)
138 if not os.path.isdir(arch_dir):
139 continue
140
141 with open(os.path.join(arch_dir, "Release"), "w+") as release:
142 release.write("Label: %s" % arch)
143
144 index_cmds.append("cd %s; %s . | %s > Packages.gz" %
145 (arch_dir, dpkg_scanpackages, gzip))
146
147 deb_dirs_found = True
148
149 if not deb_dirs_found:
150 return("There are no packages in %s" % self.deploy_dir)
151
152 nproc = multiprocessing.cpu_count()
153 pool = bb.utils.multiprocessingpool(nproc)
154 results = list(pool.imap(create_index, index_cmds))
155 pool.close()
156 pool.join()
157
158 for result in results:
159 if result is not None:
160 return(result)
161
162
25class PackageManager(object): 163class PackageManager(object):
26 """ 164 """
27 This is an abstract class. Do not instantiate this directly. 165 This is an abstract class. Do not instantiate this directly.
@@ -136,14 +274,13 @@ class RpmPM(PackageManager):
136 def __init__(self, 274 def __init__(self,
137 d, 275 d,
138 target_rootfs, 276 target_rootfs,
139 package_archs,
140 target_os,
141 target_vendor, 277 target_vendor,
142 task_name='target', 278 task_name='target',
143 providename=None): 279 providename=None,
280 arch_var=None,
281 os_var=None):
144 super(RpmPM, self).__init__(d) 282 super(RpmPM, self).__init__(d)
145 self.target_rootfs = target_rootfs 283 self.target_rootfs = target_rootfs
146 self.ml_os_list = target_os
147 self.target_vendor = target_vendor 284 self.target_vendor = target_vendor
148 self.task_name = task_name 285 self.task_name = task_name
149 self.providename = providename 286 self.providename = providename
@@ -164,19 +301,58 @@ class RpmPM(PackageManager):
164 if not os.path.exists(self.d.expand('${T}/saved')): 301 if not os.path.exists(self.d.expand('${T}/saved')):
165 bb.utils.mkdirhier(self.d.expand('${T}/saved')) 302 bb.utils.mkdirhier(self.d.expand('${T}/saved'))
166 303
167 # arch order is reversed. This ensures the -best- match is 304 self.indexer = RpmIndexer(self.d, self.deploy_dir)
168 # listed first! 305
169 self.ml_prefix_list = dict() 306 self.ml_prefix_list, self.ml_os_list = self._get_prefix_and_os_list(arch_var, os_var)
307
308 def _get_prefix_and_os_list(self, arch_var, os_var):
309 package_archs = {
310 'default': [],
311 }
312
313 target_os = {
314 'default': "",
315 }
316
317 if arch_var is not None and os_var is not None:
318 package_archs['default'] = self.d.getVar(arch_var, True).split()
319 package_archs['default'].reverse()
320 target_os['default'] = self.d.getVar(os_var, True).strip()
321 else:
322 package_archs['default'] = self.d.getVar("PACKAGE_ARCHS", True).split()
323 # arch order is reversed. This ensures the -best- match is
324 # listed first!
325 package_archs['default'].reverse()
326 target_os['default'] = self.d.getVar("TARGET_OS", True).strip()
327 multilibs = self.d.getVar('MULTILIBS', True) or ""
328 for ext in multilibs.split():
329 eext = ext.split(':')
330 if len(eext) > 1 and eext[0] == 'multilib':
331 localdata = bb.data.createCopy(self.d)
332 default_tune_key = "DEFAULTTUNE_virtclass-multilib-" + eext[1]
333 default_tune = localdata.getVar(default_tune_key, False)
334 if default_tune:
335 localdata.setVar("DEFAULTTUNE", default_tune)
336 bb.data.update_data(localdata)
337 package_archs[eext[1]] = localdata.getVar('PACKAGE_ARCHS',
338 True).split()
339 package_archs[eext[1]].reverse()
340 target_os[eext[1]] = localdata.getVar("TARGET_OS",
341 True).strip()
342
343 ml_prefix_list = dict()
170 for mlib in package_archs: 344 for mlib in package_archs:
171 if mlib == 'default': 345 if mlib == 'default':
172 self.ml_prefix_list[mlib] = package_archs[mlib] 346 ml_prefix_list[mlib] = package_archs[mlib]
173 else: 347 else:
174 self.ml_prefix_list[mlib] = list() 348 ml_prefix_list[mlib] = list()
175 for arch in package_archs[mlib]: 349 for arch in package_archs[mlib]:
176 if arch in ['all', 'noarch', 'any']: 350 if arch in ['all', 'noarch', 'any']:
177 self.ml_prefix_list[mlib].append(arch) 351 ml_prefix_list[mlib].append(arch)
178 else: 352 else:
179 self.ml_prefix_list[mlib].append(mlib + "_" + arch) 353 ml_prefix_list[mlib].append(mlib + "_" + arch)
354
355 return (ml_prefix_list, target_os)
180 356
181 ''' 357 '''
182 Create configs for rpm and smart, and multilib is supported 358 Create configs for rpm and smart, and multilib is supported
@@ -552,39 +728,10 @@ class RpmPM(PackageManager):
552 self._invoke_smart('upgrade') 728 self._invoke_smart('upgrade')
553 729
554 def write_index(self): 730 def write_index(self):
555 arch_list = set() 731 result = self.indexer.write_index()
556 for mlib in self.ml_prefix_list:
557 for arch in self.ml_prefix_list[mlib]:
558 if arch not in arch_list:
559 arch_list.add(arch.replace('-', '_'))
560 732
561 sdk_pkg_archs = (self.d.getVar('SDK_PACKAGE_ARCHS', True) or "").replace('-', '_') 733 if result is not None:
562 arch_list = arch_list.union(set(sdk_pkg_archs.split())) 734 bb.fatal(result)
563
564 rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo")
565 index_cmds = []
566 rpm_dirs_found = False
567 for arch in arch_list:
568 arch_dir = os.path.join(self.deploy_dir, arch)
569 if not os.path.isdir(arch_dir):
570 continue
571
572 index_cmds.append("%s --update -q %s" % (rpm_createrepo, arch_dir))
573
574 rpm_dirs_found = True
575
576 if not rpm_dirs_found:
577 bb.fatal("There are no packages in %s" % self.deploy_dir)
578
579 nproc = multiprocessing.cpu_count()
580 pool = bb.utils.multiprocessingpool(nproc)
581 results = list(pool.imap(create_index, index_cmds))
582 pool.close()
583 pool.join()
584
585 for result in results:
586 if result is not None:
587 bb.fatal(result)
588 735
589 def remove_packaging_data(self): 736 def remove_packaging_data(self):
590 bb.utils.remove(self.image_rpmlib, True) 737 bb.utils.remove(self.image_rpmlib, True)
@@ -810,6 +957,8 @@ class OpkgPM(PackageManager):
810 else: 957 else:
811 self._create_custom_config() 958 self._create_custom_config()
812 959
960 self.indexer = OpkgIndexer(self.d, self.deploy_dir)
961
813 """ 962 """
814 This function will change a package's status in /var/lib/opkg/status file. 963 This function will change a package's status in /var/lib/opkg/status file.
815 If 'packages' is None then the new_status will be applied to all 964 If 'packages' is None then the new_status will be applied to all
@@ -943,53 +1092,14 @@ class OpkgPM(PackageManager):
943 "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) 1092 "returned %d:\n%s" % (e.cmd, e.returncode, e.output))
944 1093
945 def write_index(self): 1094 def write_index(self):
946 arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
947 "SDK_PACKAGE_ARCHS",
948 "MULTILIB_ARCHS"]
949
950 tmpdir = self.d.getVar('TMPDIR', True)
951
952 self.deploy_dir_lock() 1095 self.deploy_dir_lock()
953 1096
954 opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index") 1097 result = self.indexer.write_index()
955
956 if not os.path.exists(os.path.join(self.deploy_dir, "Packages")):
957 open(os.path.join(self.deploy_dir, "Packages"), "w").close()
958
959 index_cmds = []
960 for arch_var in arch_vars:
961 archs = self.d.getVar(arch_var, True)
962 if archs is None:
963 continue
964
965 for arch in archs.split():
966 pkgs_dir = os.path.join(self.deploy_dir, arch)
967 pkgs_file = os.path.join(pkgs_dir, "Packages")
968
969 if not os.path.isdir(pkgs_dir):
970 continue
971
972 if not os.path.exists(pkgs_file):
973 open(pkgs_file, "w").close()
974
975 index_cmds.append('%s -r %s -p %s -m %s' %
976 (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir))
977
978 if len(index_cmds) == 0:
979 self.deploy_dir_unlock()
980 bb.fatal("There are no packages in %s!" % self.deploy_dir)
981
982 nproc = multiprocessing.cpu_count()
983 pool = bb.utils.multiprocessingpool(nproc)
984 results = list(pool.imap(create_index, index_cmds))
985 pool.close()
986 pool.join()
987 1098
988 self.deploy_dir_unlock() 1099 self.deploy_dir_unlock()
989 1100
990 for result in results: 1101 if result is not None:
991 if result is not None: 1102 bb.fatal(result)
992 bb.fatal(result)
993 1103
994 def remove_packaging_data(self): 1104 def remove_packaging_data(self):
995 bb.utils.remove(self.opkg_dir, True) 1105 bb.utils.remove(self.opkg_dir, True)
@@ -1078,6 +1188,8 @@ class DpkgPM(PackageManager):
1078 1188
1079 self._create_configs(archs, base_archs) 1189 self._create_configs(archs, base_archs)
1080 1190
1191 self.indexer = DpkgIndexer(self.d, self.deploy_dir)
1192
1081 """ 1193 """
1082 This function will change a package's status in /var/lib/dpkg/status file. 1194 This function will change a package's status in /var/lib/dpkg/status file.
1083 If 'packages' is None then the new_status will be applied to all 1195 If 'packages' is None then the new_status will be applied to all
@@ -1215,49 +1327,14 @@ class DpkgPM(PackageManager):
1215 "returned %d:\n%s" % (e.cmd, e.returncode, e.output)) 1327 "returned %d:\n%s" % (e.cmd, e.returncode, e.output))
1216 1328
1217 def write_index(self): 1329 def write_index(self):
1218 tmpdir = self.d.getVar('TMPDIR', True)
1219
1220 pkg_archs = self.d.getVar('PACKAGE_ARCHS', True)
1221 if pkg_archs is not None:
1222 arch_list = pkg_archs.split()
1223 sdk_pkg_archs = self.d.getVar('SDK_PACKAGE_ARCHS', True)
1224 if sdk_pkg_archs is not None:
1225 arch_list += sdk_pkg_archs.split()
1226
1227 dpkg_scanpackages = bb.utils.which(os.getenv('PATH'), "dpkg-scanpackages")
1228 gzip = bb.utils.which(os.getenv('PATH'), "gzip")
1229
1230 self.deploy_dir_lock() 1330 self.deploy_dir_lock()
1231 1331
1232 index_cmds = [] 1332 result = self.indexer.write_index()
1233 deb_dirs_found = False
1234 for arch in arch_list:
1235 arch_dir = os.path.join(self.deploy_dir, arch)
1236 if not os.path.isdir(arch_dir):
1237 continue
1238
1239 with open(os.path.join(arch_dir, "Release"), "w+") as release:
1240 release.write("Label: %s" % arch)
1241
1242 index_cmds.append("cd %s; %s . | %s > Packages.gz" %
1243 (arch_dir, dpkg_scanpackages, gzip))
1244
1245 deb_dirs_found = True
1246
1247 if not deb_dirs_found:
1248 bb.fatal("There are no packages in %s" % self.deploy_dir)
1249
1250 nproc = multiprocessing.cpu_count()
1251 pool = bb.utils.multiprocessingpool(nproc)
1252 results = list(pool.imap(create_index, index_cmds))
1253 pool.close()
1254 pool.join()
1255 1333
1256 self.deploy_dir_unlock() 1334 self.deploy_dir_unlock()
1257 1335
1258 for result in results: 1336 if result is not None:
1259 if result is not None: 1337 bb.fatal(result)
1260 bb.fatal(result)
1261 1338
1262 def _create_configs(self, archs, base_archs): 1339 def _create_configs(self, archs, base_archs):
1263 base_archs = re.sub("_", "-", base_archs) 1340 base_archs = re.sub("_", "-", base_archs)
@@ -1365,6 +1442,22 @@ class DpkgPM(PackageManager):
1365 1442
1366 return output 1443 return output
1367 1444
1445
1446def generate_index_files(d):
1447 img_type = d.getVar('IMAGE_PKGTYPE', True)
1448
1449 result = None
1450
1451 if img_type == "rpm":
1452 result = RpmIndexer(d, d.getVar('DEPLOY_DIR_RPM', True)).write_index()
1453 elif img_type == "ipk":
1454 result = OpkgIndexer(d, d.getVar('DEPLOY_DIR_IPK', True)).write_index()
1455 elif img_type == "deb":
1456 result = DpkgIndexer(d, d.getVar('DEPLOY_DIR_DEB', True)).write_index()
1457
1458 if result is not None:
1459 bb.fatal(result)
1460
1368if __name__ == "__main__": 1461if __name__ == "__main__":
1369 """ 1462 """
1370 We should be able to run this as a standalone script, from outside bitbake 1463 We should be able to run this as a standalone script, from outside bitbake