summaryrefslogtreecommitdiffstats
path: root/scripts/lib/recipetool/append.py
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2015-05-18 16:15:07 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-05-20 21:41:04 +0100
commitfbfc06a969200e582a059c9943e6fd17aca70e30 (patch)
tree1862fb721f550d42f10e8867224712754a865d34 /scripts/lib/recipetool/append.py
parentc63adf5c5b4b5984c315e914a7d3cb4b51040602 (diff)
downloadpoky-fbfc06a969200e582a059c9943e6fd17aca70e30.tar.gz
recipetool: add appendfile subcommand
Locating which recipe provides a file in an image that you want to modify and then figuring out how to bbappend the recipe in order to replace it can be a tedious process. Thus, add a new appendfile subcommand to recipetool, providing the ability to create a bbappend file to add/replace any file in the target system. Without the -r option, it will search for the recipe packaging the specified file (using pkgdata from previously built recipes). The bbappend will be created at the appropriate path within the specified layer directory (which may or may not be in your bblayers.conf) or if one already exists it will be updated appropriately. Fairly extensive oe-selftest tests are also provided. Implements [YOCTO #6447]. (From OE-Core rev: dd2aa93b3c13d2c6464ef0fda59620c7dba450bb) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/recipetool/append.py')
-rw-r--r--scripts/lib/recipetool/append.py360
1 files changed, 360 insertions, 0 deletions
diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py
new file mode 100644
index 0000000000..39117c1f66
--- /dev/null
+++ b/scripts/lib/recipetool/append.py
@@ -0,0 +1,360 @@
1# Recipe creation tool - append plugin
2#
3# Copyright (C) 2015 Intel Corporation
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
18import sys
19import os
20import argparse
21import glob
22import fnmatch
23import re
24import subprocess
25import logging
26import stat
27import shutil
28import scriptutils
29import errno
30from collections import defaultdict
31
32logger = logging.getLogger('recipetool')
33
34tinfoil = None
35
36def plugin_init(pluginlist):
37 # Don't need to do anything here right now, but plugins must have this function defined
38 pass
39
40def tinfoil_init(instance):
41 global tinfoil
42 tinfoil = instance
43
44
45# FIXME guessing when we don't have pkgdata?
46# FIXME mode to create patch rather than directly substitute
47
48class InvalidTargetFileError(Exception):
49 pass
50
51def find_target_file(targetpath, d, pkglist=None):
52 """Find the recipe installing the specified target path, optionally limited to a select list of packages"""
53 import json
54
55 pkgdata_dir = d.getVar('PKGDATA_DIR', True)
56
57 # The mix between /etc and ${sysconfdir} here may look odd, but it is just
58 # being consistent with usage elsewhere
59 invalidtargets = {'${sysconfdir}/version': '${sysconfdir}/version is written out at image creation time',
60 '/etc/timestamp': '/etc/timestamp is written out at image creation time',
61 '/dev/*': '/dev is handled by udev (or equivalent) and the kernel (devtmpfs)',
62 '/etc/passwd': '/etc/passwd should be managed through the useradd and extrausers classes',
63 '/etc/group': '/etc/group should be managed through the useradd and extrausers classes',
64 '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes',
65 '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes',
66 '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname_pn-base-files = "value" in configuration',}
67
68 for pthspec, message in invalidtargets.iteritems():
69 if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)):
70 raise InvalidTargetFileError(d.expand(message))
71
72 targetpath_re = re.compile(r'\s+(\$D)?%s(\s|$)' % targetpath)
73
74 recipes = defaultdict(list)
75 for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')):
76 if pkglist:
77 filelist = pkglist
78 else:
79 filelist = files
80 for fn in filelist:
81 pkgdatafile = os.path.join(root, fn)
82 if pkglist and not os.path.exists(pkgdatafile):
83 continue
84 with open(pkgdatafile, 'r') as f:
85 pn = ''
86 # This does assume that PN comes before other values, but that's a fairly safe assumption
87 for line in f:
88 if line.startswith('PN:'):
89 pn = line.split(':', 1)[1].strip()
90 elif line.startswith('FILES_INFO:'):
91 val = line.split(':', 1)[1].strip()
92 dictval = json.loads(val)
93 for fullpth in dictval.keys():
94 if fnmatch.fnmatchcase(fullpth, targetpath):
95 recipes[targetpath].append(pn)
96 elif line.startswith('pkg_preinst_') or line.startswith('pkg_postinst_'):
97 scriptval = line.split(':', 1)[1].strip().decode('string_escape')
98 if 'update-alternatives --install %s ' % targetpath in scriptval:
99 recipes[targetpath].append('?%s' % pn)
100 elif targetpath_re.search(scriptval):
101 recipes[targetpath].append('!%s' % pn)
102 return recipes
103
104def _get_recipe_file(cooker, pn):
105 import oe.recipeutils
106 recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
107 if not recipefile:
108 skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
109 if skipreasons:
110 logger.error('\n'.join(skipreasons))
111 else:
112 logger.error("Unable to find any recipe file matching %s" % pn)
113 return recipefile
114
115def _parse_recipe(pn, tinfoil):
116 import oe.recipeutils
117 recipefile = _get_recipe_file(tinfoil.cooker, pn)
118 if not recipefile:
119 # Error already logged
120 return None
121 append_files = tinfoil.cooker.collection.get_file_appends(recipefile)
122 rd = oe.recipeutils.parse_recipe(recipefile, append_files,
123 tinfoil.config_data)
124 return rd
125
126def determine_file_source(targetpath, rd):
127 """Assuming we know a file came from a specific recipe, figure out exactly where it came from"""
128 import oe.recipeutils
129
130 # See if it's in do_install for the recipe
131 workdir = rd.getVar('WORKDIR', True)
132 src_uri = rd.getVar('SRC_URI', True)
133 srcfile = ''
134 modpatches = []
135 elements = check_do_install(rd, targetpath)
136 if elements:
137 logger.debug('do_install line:\n%s' % ' '.join(elements))
138 srcpath = get_source_path(elements)
139 logger.debug('source path: %s' % srcpath)
140 if not srcpath.startswith('/'):
141 # Handle non-absolute path
142 srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs', True).split()[-1], srcpath))
143 if srcpath.startswith(workdir):
144 # OK, now we have the source file name, look for it in SRC_URI
145 workdirfile = os.path.relpath(srcpath, workdir)
146 # FIXME this is where we ought to have some code in the fetcher, because this is naive
147 for item in src_uri.split():
148 localpath = bb.fetch2.localpath(item, rd)
149 # Source path specified in do_install might be a glob
150 if fnmatch.fnmatch(os.path.basename(localpath), workdirfile):
151 srcfile = 'file://%s' % localpath
152 elif '/' in workdirfile:
153 if item == 'file://%s' % workdirfile:
154 srcfile = 'file://%s' % localpath
155
156 # Check patches
157 srcpatches = []
158 patchedfiles = oe.recipeutils.get_recipe_patched_files(rd)
159 for patch, filelist in patchedfiles.iteritems():
160 for fileitem in filelist:
161 if fileitem[0] == srcpath:
162 srcpatches.append((patch, fileitem[1]))
163 if srcpatches:
164 addpatch = None
165 for patch in srcpatches:
166 if patch[1] == 'A':
167 addpatch = patch[0]
168 else:
169 modpatches.append(patch[0])
170 if addpatch:
171 srcfile = 'patch://%s' % addpatch
172
173 return (srcfile, elements, modpatches)
174
175def get_source_path(cmdelements):
176 """Find the source path specified within a command"""
177 command = cmdelements[0]
178 if command in ['install', 'cp']:
179 helptext = subprocess.check_output('LC_ALL=C %s --help' % command, shell=True)
180 argopts = ''
181 argopt_line_re = re.compile('^-([a-zA-Z0-9]), --[a-z-]+=')
182 for line in helptext.splitlines():
183 line = line.lstrip()
184 res = argopt_line_re.search(line)
185 if res:
186 argopts += res.group(1)
187 if not argopts:
188 # Fallback
189 if command == 'install':
190 argopts = 'gmoSt'
191 elif command == 'cp':
192 argopts = 't'
193 else:
194 raise Exception('No fallback arguments for command %s' % command)
195
196 skipnext = False
197 for elem in cmdelements[1:-1]:
198 if elem.startswith('-'):
199 if len(elem) > 1 and elem[1] in argopts:
200 skipnext = True
201 continue
202 if skipnext:
203 skipnext = False
204 continue
205 return elem
206 else:
207 raise Exception('get_source_path: no handling for command "%s"')
208
209def get_func_deps(func, d):
210 """Find the function dependencies of a shell function"""
211 deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True))
212 deps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
213 funcdeps = []
214 for dep in deps:
215 if d.getVarFlag(dep, 'func', True):
216 funcdeps.append(dep)
217 return funcdeps
218
219def check_do_install(rd, targetpath):
220 """Look at do_install for a command that installs/copies the specified target path"""
221 instpath = os.path.abspath(os.path.join(rd.getVar('D', True), targetpath.lstrip('/')))
222 do_install = rd.getVar('do_install', True)
223 # Handle where do_install calls other functions (somewhat crudely, but good enough for this purpose)
224 deps = get_func_deps('do_install', rd)
225 for dep in deps:
226 do_install = do_install.replace(dep, rd.getVar(dep, True))
227
228 # Look backwards through do_install as we want to catch where a later line (perhaps
229 # from a bbappend) is writing over the top
230 for line in reversed(do_install.splitlines()):
231 line = line.strip()
232 if (line.startswith('install ') and ' -m' in line) or line.startswith('cp '):
233 elements = line.split()
234 destpath = os.path.abspath(elements[-1])
235 if destpath == instpath:
236 return elements
237 elif destpath.rstrip('/') == os.path.dirname(instpath):
238 # FIXME this doesn't take recursive copy into account; unsure if it's practical to do so
239 srcpath = get_source_path(elements)
240 if fnmatch.fnmatchcase(os.path.basename(instpath), os.path.basename(srcpath)):
241 return elements
242 return None
243
244
245def appendfile(args):
246 import oe.recipeutils
247
248 if not args.targetpath.startswith('/'):
249 logger.error('Target path should start with /')
250 return 2
251
252 if os.path.isdir(args.newfile):
253 logger.error('Specified new file "%s" is a directory' % args.newfile)
254 return 2
255
256 if not os.path.exists(args.destlayer):
257 logger.error('Destination layer directory "%s" does not exist' % args.destlayer)
258 return 2
259 if not os.path.exists(os.path.join(args.destlayer, 'conf', 'layer.conf')):
260 logger.error('conf/layer.conf not found in destination layer "%s"' % args.destlayer)
261 return 2
262
263 stdout = ''
264 try:
265 (stdout, _) = bb.process.run('LANG=C file -E -b %s' % args.newfile, shell=True)
266 except bb.process.ExecutionError as err:
267 logger.debug('file command returned error: %s' % err)
268 pass
269 if stdout:
270 logger.debug('file command output: %s' % stdout.rstrip())
271 if ('executable' in stdout and not 'shell script' in stdout) or 'shared object' in stdout:
272 logger.warn('This file looks like it is a binary or otherwise the output of compilation. If it is, you should consider building it properly instead of substituting a binary file directly.')
273
274 if args.recipe:
275 recipes = {args.targetpath: [args.recipe],}
276 else:
277 try:
278 recipes = find_target_file(args.targetpath, tinfoil.config_data)
279 except InvalidTargetFileError as e:
280 logger.error('%s cannot be handled by this tool: %s' % (args.targetpath, e))
281 return 1
282 if not recipes:
283 logger.error('Unable to find any package producing path %s - this may be because the recipe packaging it has not been built yet' % args.targetpath)
284 return 1
285
286 alternative_pns = []
287 postinst_pns = []
288
289 selectpn = None
290 for targetpath, pnlist in recipes.iteritems():
291 for pn in pnlist:
292 if pn.startswith('?'):
293 alternative_pns.append(pn[1:])
294 elif pn.startswith('!'):
295 postinst_pns.append(pn[1:])
296 else:
297 selectpn = pn
298
299 if not selectpn and len(alternative_pns) == 1:
300 selectpn = alternative_pns[0]
301 logger.error('File %s is an alternative possibly provided by recipe %s but seemingly no other, selecting it by default - you should double check other recipes' % (args.targetpath, selectpn))
302
303 if selectpn:
304 logger.debug('Selecting recipe %s for file %s' % (selectpn, args.targetpath))
305 if postinst_pns:
306 logger.warn('%s be modified by postinstall scripts for the following recipes:\n %s\nThis may or may not be an issue depending on what modifications these postinstall scripts make.' % (args.targetpath, '\n '.join(postinst_pns)))
307 rd = _parse_recipe(selectpn, tinfoil)
308 if not rd:
309 # Error message already shown
310 return 1
311 sourcefile, instelements, modpatches = determine_file_source(args.targetpath, rd)
312 sourcepath = None
313 if sourcefile:
314 sourcetype, sourcepath = sourcefile.split('://', 1)
315 logger.debug('Original source file is %s (%s)' % (sourcepath, sourcetype))
316 if sourcetype == 'patch':
317 logger.warn('File %s is added by the patch %s - you may need to remove or replace this patch in order to replace the file.' % (args.targetpath, sourcepath))
318 sourcepath = None
319 else:
320 logger.debug('Unable to determine source file, proceeding anyway')
321 if modpatches:
322 logger.warn('File %s is modified by the following patches:\n %s' % (args.targetpath, '\n '.join(modpatches)))
323
324 if instelements and sourcepath:
325 install = None
326 else:
327 # Auto-determine permissions
328 # Check destination
329 binpaths = '${bindir}:${sbindir}:${base_bindir}:${base_sbindir}:${libexecdir}:${sysconfdir}/init.d'
330 perms = '0644'
331 if os.path.abspath(os.path.dirname(args.targetpath)) in rd.expand(binpaths).split(':'):
332 # File is going into a directory normally reserved for executables, so it should be executable
333 perms = '0755'
334 else:
335 # Check source
336 st = os.stat(args.newfile)
337 if st.st_mode & stat.S_IXUSR:
338 perms = '0755'
339 install = {args.newfile: (args.targetpath, perms)}
340 oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine)
341 return 0
342 else:
343 if alternative_pns:
344 logger.error('File %s is an alternative possibly provided by the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(alternative_pns)))
345 elif postinst_pns:
346 logger.error('File %s may be written out in a pre/postinstall script of the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(postinst_pns)))
347 return 3
348
349
350def register_command(subparsers):
351 parser_appendfile = subparsers.add_parser('appendfile',
352 help='Create a bbappend to replace a file',
353 description='')
354 parser_appendfile.add_argument('destlayer', help='Destination layer to write the bbappend to')
355 parser_appendfile.add_argument('targetpath', help='Path within the image to the file to be replaced')
356 parser_appendfile.add_argument('newfile', help='Custom file to replace it with')
357 parser_appendfile.add_argument('-r', '--recipe', help='Override recipe to apply to (default is to find which recipe already packages it)')
358 parser_appendfile.add_argument('-m', '--machine', help='Make bbappend changes specific to a machine only', metavar='MACHINE')
359 parser_appendfile.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true')
360 parser_appendfile.set_defaults(func=appendfile, parserecipes=True)