summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-06-06 19:33:02 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-06-07 09:11:49 +0100
commit1334b064d3f04a976a42fe9dad64831b33d93755 (patch)
treefb273b3e4355c2ca5d8a14d4ed7905fa56223fa8
parent0a72ec2ed1fd0d1bd7951c1d21be731c873e0964 (diff)
downloadpoky-1334b064d3f04a976a42fe9dad64831b33d93755.tar.gz
gpg_sign/selftest: Fix secmem parameter handling
We keep seeing "cannot allocate memory" errors from rpm when signing packages on the autobuilder. The following were tried: * checking locked memory use (isn't hitting limits) * Restricting RPM_GPG_SIGN_CHUNK to 1 * Limiting to 10 parallel do_package_write_rpm tasks * Allowing unlimied memory overcommit * Disabling rpm parallel compression and the test still failed. Further invetigation showed that the --auto-expand-secmem wasn't being passed to gpg-agent which meant the secmem couldn't be expanded hence the errors when there was pressure on the agent. The reason this happens is that some of the early gpg commands can start the agent without the option and it sticks around in memory so a version with the correct option may or may not get started. We therefore add the option to all the key gpg calls. (From OE-Core rev: c7e131a76e522503df55e211dd261829feacfa28) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/lib/oe/gpg_sign.py39
-rw-r--r--meta/lib/oeqa/selftest/cases/signing.py3
2 files changed, 24 insertions, 18 deletions
diff --git a/meta/lib/oe/gpg_sign.py b/meta/lib/oe/gpg_sign.py
index a95d2ba34c..2fd8c3b1ac 100644
--- a/meta/lib/oe/gpg_sign.py
+++ b/meta/lib/oe/gpg_sign.py
@@ -15,21 +15,27 @@ class LocalSigner(object):
15 def __init__(self, d): 15 def __init__(self, d):
16 self.gpg_bin = d.getVar('GPG_BIN') or \ 16 self.gpg_bin = d.getVar('GPG_BIN') or \
17 bb.utils.which(os.getenv('PATH'), 'gpg') 17 bb.utils.which(os.getenv('PATH'), 'gpg')
18 self.gpg_cmd = [self.gpg_bin]
19 self.gpg_agent_bin = bb.utils.which(os.getenv('PATH'), "gpg-agent")
20 # Without this we see "Cannot allocate memory" errors when running processes in parallel
21 # It needs to be set for any gpg command since any agent launched can stick around in memory
22 # and this parameter must be set.
23 if self.gpg_agent_bin:
24 self.gpg_cmd += ["--agent-program=%s|--auto-expand-secmem" % (self.gpg_agent_bin)]
18 self.gpg_path = d.getVar('GPG_PATH') 25 self.gpg_path = d.getVar('GPG_PATH')
19 self.gpg_version = self.get_gpg_version()
20 self.rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmsign") 26 self.rpm_bin = bb.utils.which(os.getenv('PATH'), "rpmsign")
21 self.gpg_agent_bin = bb.utils.which(os.getenv('PATH'), "gpg-agent") 27 self.gpg_version = self.get_gpg_version()
28
22 29
23 def export_pubkey(self, output_file, keyid, armor=True): 30 def export_pubkey(self, output_file, keyid, armor=True):
24 """Export GPG public key to a file""" 31 """Export GPG public key to a file"""
25 cmd = '%s --no-permission-warning --batch --yes --export -o %s ' % \ 32 cmd = self.gpg_cmd + ["--no-permission-warning", "--batch", "--yes", "--export", "-o", output_file]
26 (self.gpg_bin, output_file)
27 if self.gpg_path: 33 if self.gpg_path:
28 cmd += "--homedir %s " % self.gpg_path 34 cmd += ["--homedir", self.gpg_path]
29 if armor: 35 if armor:
30 cmd += "--armor " 36 cmd += ["--armor"]
31 cmd += keyid 37 cmd += [keyid]
32 subprocess.check_output(shlex.split(cmd), stderr=subprocess.STDOUT) 38 subprocess.check_output(cmd, stderr=subprocess.STDOUT)
33 39
34 def sign_rpms(self, files, keyid, passphrase, digest, sign_chunk, fsk=None, fsk_password=None): 40 def sign_rpms(self, files, keyid, passphrase, digest, sign_chunk, fsk=None, fsk_password=None):
35 """Sign RPM files""" 41 """Sign RPM files"""
@@ -59,7 +65,7 @@ class LocalSigner(object):
59 if passphrase_file and passphrase: 65 if passphrase_file and passphrase:
60 raise Exception("You should use either passphrase_file of passphrase, not both") 66 raise Exception("You should use either passphrase_file of passphrase, not both")
61 67
62 cmd = [self.gpg_bin, '--detach-sign', '--no-permission-warning', '--batch', 68 cmd = self.gpg_cmd + ['--detach-sign', '--no-permission-warning', '--batch',
63 '--no-tty', '--yes', '--passphrase-fd', '0', '-u', keyid] 69 '--no-tty', '--yes', '--passphrase-fd', '0', '-u', keyid]
64 70
65 if self.gpg_path: 71 if self.gpg_path:
@@ -72,9 +78,6 @@ class LocalSigner(object):
72 if self.gpg_version > (2,1,): 78 if self.gpg_version > (2,1,):
73 cmd += ['--pinentry-mode', 'loopback'] 79 cmd += ['--pinentry-mode', 'loopback']
74 80
75 if self.gpg_agent_bin:
76 cmd += ["--agent-program=%s|--auto-expand-secmem" % (self.gpg_agent_bin)]
77
78 cmd += [input_file] 81 cmd += [input_file]
79 82
80 try: 83 try:
@@ -101,7 +104,8 @@ class LocalSigner(object):
101 def get_gpg_version(self): 104 def get_gpg_version(self):
102 """Return the gpg version as a tuple of ints""" 105 """Return the gpg version as a tuple of ints"""
103 try: 106 try:
104 ver_str = subprocess.check_output((self.gpg_bin, "--version", "--no-permission-warning")).split()[2].decode("utf-8") 107 cmd = self.gpg_cmd + ["--version", "--no-permission-warning"]
108 ver_str = subprocess.check_output(cmd).split()[2].decode("utf-8")
105 return tuple([int(i) for i in ver_str.split("-")[0].split('.')]) 109 return tuple([int(i) for i in ver_str.split("-")[0].split('.')])
106 except subprocess.CalledProcessError as e: 110 except subprocess.CalledProcessError as e:
107 raise bb.build.FuncFailed("Could not get gpg version: %s" % e) 111 raise bb.build.FuncFailed("Could not get gpg version: %s" % e)
@@ -109,11 +113,12 @@ class LocalSigner(object):
109 113
110 def verify(self, sig_file): 114 def verify(self, sig_file):
111 """Verify signature""" 115 """Verify signature"""
112 cmd = self.gpg_bin + " --verify --no-permission-warning " 116 cmd = self.gpg_cmd + [" --verify", "--no-permission-warning"]
113 if self.gpg_path: 117 if self.gpg_path:
114 cmd += "--homedir %s " % self.gpg_path 118 cmd += ["--homedir", self.gpg_path]
115 cmd += sig_file 119
116 status = subprocess.call(shlex.split(cmd)) 120 cmd += [sig_file]
121 status = subprocess.call(cmd)
117 ret = False if status else True 122 ret = False if status else True
118 return ret 123 return ret
119 124
diff --git a/meta/lib/oeqa/selftest/cases/signing.py b/meta/lib/oeqa/selftest/cases/signing.py
index 9c710bd0ff..b390f37d8e 100644
--- a/meta/lib/oeqa/selftest/cases/signing.py
+++ b/meta/lib/oeqa/selftest/cases/signing.py
@@ -30,7 +30,8 @@ class Signing(OESelftestTestCase):
30 self.secret_key_path = os.path.join(self.testlayer_path, 'files', 'signing', "key.secret") 30 self.secret_key_path = os.path.join(self.testlayer_path, 'files', 'signing', "key.secret")
31 31
32 nsysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "gnupg-native") 32 nsysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "gnupg-native")
33 runCmd('gpg --batch --homedir %s --import %s %s' % (self.gpg_dir, self.pub_key_path, self.secret_key_path), native_sysroot=nsysroot) 33
34 runCmd('gpg --agent-program=`which gpg-agent`\|--auto-expand-secmem --batch --homedir %s --import %s %s' % (self.gpg_dir, self.pub_key_path, self.secret_key_path), native_sysroot=nsysroot)
34 return nsysroot + get_bb_var("bindir_native") 35 return nsysroot + get_bb_var("bindir_native")
35 36
36 37