summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorStephano Cetola <stephano.cetola@linux.intel.com>2016-08-10 20:03:16 (GMT)
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-08-12 14:25:22 (GMT)
commit6b66e9317f4ec3a69f98f29836aafa35b52f3fc7 (patch)
treee97a5f491be57a28ebbb334af7fc706e93884904
parentd11e8e1109c2c5a0cc3e4dea70cebe068b2a6ba1 (diff)
downloadpoky-6b66e9317f4ec3a69f98f29836aafa35b52f3fc7.tar.gz
Allow for simultaneous do_rootfs tasks with rpmuninative-1.3
Give each rootfs its own RPM channel to use. This puts the RPM metadata in a private subdirectory of $WORKDIR, rather than living in DEPLOY_DIR where other tasks may race with it. This allows us to reduce the time that the rpm.lock is held to only the time needed to hardlink the RPMs, allowing the majority of the rootfs operation to run in parallel. Also, this fixes the smart tests by generating an index for all packages at the time of the test, rather than using the one provided by the rootfs process. Original credit for the enhancement should go to Steven Walter stevenrwalter@gmail.com. (From OE-Core rev: a92c196449c516fe51786d429078bbb1213bb029) Signed-off-by: Stephano Cetola <stephano.cetola@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/rootfs_rpm.bbclass5
-rw-r--r--meta/lib/oe/package_manager.py17
-rw-r--r--meta/lib/oeqa/runtime/smart.py41
3 files changed, 54 insertions, 9 deletions
diff --git a/meta/classes/rootfs_rpm.bbclass b/meta/classes/rootfs_rpm.bbclass
index 38b3c99..37730a7 100644
--- a/meta/classes/rootfs_rpm.bbclass
+++ b/meta/classes/rootfs_rpm.bbclass
@@ -24,11 +24,6 @@ do_populate_sdk[depends] += "${RPMROOTFSDEPENDS}"
24do_rootfs[recrdeptask] += "do_package_write_rpm" 24do_rootfs[recrdeptask] += "do_package_write_rpm"
25do_rootfs[vardeps] += "PACKAGE_FEED_URIS" 25do_rootfs[vardeps] += "PACKAGE_FEED_URIS"
26 26
27# RPM doesn't work with multiple rootfs generation at once due to collisions in the use of files
28# in ${DEPLOY_DIR_RPM}. This can be removed if package_update_index_rpm can be called concurrently
29do_rootfs[lockfiles] += "${DEPLOY_DIR_RPM}/rpm.lock"
30do_populate_sdk[lockfiles] += "${DEPLOY_DIR_RPM}/rpm.lock"
31
32python () { 27python () {
33 if d.getVar('BUILD_IMAGES_FROM_FEEDS', True): 28 if d.getVar('BUILD_IMAGES_FROM_FEEDS', True):
34 flags = d.getVarFlag('do_rootfs', 'recrdeptask', True) 29 flags = d.getVarFlag('do_rootfs', 'recrdeptask', True)
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 47f6831..2802254 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -9,6 +9,7 @@ import collections
9import bb 9import bb
10import tempfile 10import tempfile
11import oe.utils 11import oe.utils
12import oe.path
12import string 13import string
13from oe.gpg_sign import get_signer 14from oe.gpg_sign import get_signer
14 15
@@ -175,7 +176,7 @@ class RpmIndexer(Indexer):
175 dbpath = os.path.join(self.d.getVar('WORKDIR', True), 'rpmdb', arch) 176 dbpath = os.path.join(self.d.getVar('WORKDIR', True), 'rpmdb', arch)
176 if os.path.exists(dbpath): 177 if os.path.exists(dbpath):
177 bb.utils.remove(dbpath, True) 178 bb.utils.remove(dbpath, True)
178 arch_dir = os.path.join(self.deploy_dir, arch) 179 arch_dir = os.path.join(self.d.getVar('WORKDIR', True), 'rpms', arch)
179 if not os.path.isdir(arch_dir): 180 if not os.path.isdir(arch_dir):
180 continue 181 continue
181 182
@@ -1010,8 +1011,18 @@ class RpmPM(PackageManager):
1010 ch_already_added = [] 1011 ch_already_added = []
1011 for canonical_arch in platform_extra: 1012 for canonical_arch in platform_extra:
1012 arch = canonical_arch.split('-')[0] 1013 arch = canonical_arch.split('-')[0]
1013 arch_channel = os.path.join(self.deploy_dir, arch) 1014 arch_channel = os.path.join(self.d.getVar('WORKDIR', True), 'rpms', arch)
1014 if os.path.exists(arch_channel) and not arch in ch_already_added: 1015 oe.path.remove(arch_channel)
1016 deploy_arch_dir = os.path.join(self.deploy_dir, arch)
1017 if not os.path.exists(deploy_arch_dir):
1018 continue
1019
1020 lockfilename = self.d.getVar('DEPLOY_DIR_RPM', True) + "/rpm.lock"
1021 lf = bb.utils.lockfile(lockfilename, False)
1022 oe.path.copyhardlinktree(deploy_arch_dir, arch_channel)
1023 bb.utils.unlockfile(lf)
1024
1025 if not arch in ch_already_added:
1015 bb.note('Adding Smart channel %s (%s)' % 1026 bb.note('Adding Smart channel %s (%s)' %
1016 (arch, channel_priority)) 1027 (arch, channel_priority))
1017 self._invoke_smart('channel --add %s type=rpm-md baseurl=%s -y' 1028 self._invoke_smart('channel --add %s type=rpm-md baseurl=%s -y'
diff --git a/meta/lib/oeqa/runtime/smart.py b/meta/lib/oeqa/runtime/smart.py
index c7a5753..c8ba433 100644
--- a/meta/lib/oeqa/runtime/smart.py
+++ b/meta/lib/oeqa/runtime/smart.py
@@ -1,5 +1,7 @@
1import unittest 1import unittest
2import re 2import re
3import oe
4import subprocess
3from oeqa.oetest import oeRuntimeTest, skipModule 5from oeqa.oetest import oeRuntimeTest, skipModule
4from oeqa.utils.decorators import * 6from oeqa.utils.decorators import *
5from oeqa.utils.httpserver import HTTPService 7from oeqa.utils.httpserver import HTTPService
@@ -53,9 +55,46 @@ class SmartBasicTest(SmartTest):
53class SmartRepoTest(SmartTest): 55class SmartRepoTest(SmartTest):
54 56
55 @classmethod 57 @classmethod
58 def create_index(self, arg):
59 index_cmd = arg
60 try:
61 bb.note("Executing '%s' ..." % index_cmd)
62 result = subprocess.check_output(index_cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8")
63 except subprocess.CalledProcessError as e:
64 return("Index creation command '%s' failed with return code %d:\n%s" %
65 (e.cmd, e.returncode, e.output.decode("utf-8")))
66 if result:
67 bb.note(result)
68 return None
69
70 @classmethod
56 def setUpClass(self): 71 def setUpClass(self):
57 self.repolist = [] 72 self.repolist = []
58 self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR', True), oeRuntimeTest.tc.target.server_ip) 73
74 # Index RPMs
75 rpm_createrepo = bb.utils.which(os.getenv('PATH'), "createrepo")
76 index_cmds = []
77 rpm_dirs_found = False
78 archs = (oeRuntimeTest.tc.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS', True) or "").replace('-', '_').split()
79 for arch in archs:
80 rpm_dir = os.path.join(oeRuntimeTest.tc.d.getVar('DEPLOY_DIR_RPM', True), arch)
81 idx_path = os.path.join(oeRuntimeTest.tc.d.getVar('WORKDIR', True), 'rpm', arch)
82 db_path = os.path.join(oeRuntimeTest.tc.d.getVar('WORKDIR', True), 'rpmdb', arch)
83 if not os.path.isdir(rpm_dir):
84 continue
85 if os.path.exists(db_path):
86 bb.utils.remove(dbpath, True)
87 lockfilename = oeRuntimeTest.tc.d.getVar('DEPLOY_DIR_RPM', True) + "/rpm.lock"
88 lf = bb.utils.lockfile(lockfilename, False)
89 oe.path.copyhardlinktree(rpm_dir, idx_path)
90 bb.utils.unlockfile(lf)
91 index_cmds.append("%s --dbpath %s --update -q %s" % (rpm_createrepo, db_path, idx_path))
92 rpm_dirs_found = True
93 # Create repodata¬
94 result = oe.utils.multiprocess_exec(index_cmds, self.create_index)
95 if result:
96 bb.fatal('%s' % ('\n'.join(result)))
97 self.repo_server = HTTPService(oeRuntimeTest.tc.d.getVar('WORKDIR', True), oeRuntimeTest.tc.target.server_ip)
59 self.repo_server.start() 98 self.repo_server.start()
60 99
61 @classmethod 100 @classmethod