summaryrefslogtreecommitdiffstats
path: root/scripts/lib
diff options
context:
space:
mode:
authorLeonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>2015-09-08 11:39:09 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-09-09 14:27:51 +0100
commit8be95c5fbe6ad970943edabe288fd47d1dcac288 (patch)
treea3a658ff9b01a28d607aa68ea08789fcc666bbed /scripts/lib
parent7cde0ebd59f12da363e780126ca4284e0930721d (diff)
downloadpoky-8be95c5fbe6ad970943edabe288fd47d1dcac288.tar.gz
devtool: add upgrade feature
Upgrades a recipe to a particular version and downloads the source code into a folder. User can avoid patching the source code. These are the general steps of the upgrade function: - Extract current recipe source code into srctree and create a branch - Extract upgrade recipe source code into srctree and rebase with previous branch. In case the rebase is not correctly applied, source code will not be deleted, so user correct the patches - Creates the new recipe under the workspace [YOCTO #7642] (From OE-Core rev: 4020f5d91b3e4d011150d5081d36215f8eab732e) Signed-off-by: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com> Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib')
-rw-r--r--scripts/lib/devtool/standard.py4
-rw-r--r--scripts/lib/devtool/upgrade.py354
2 files changed, 357 insertions, 1 deletions
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index a5a20e3ae8..4d51a458fe 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -178,6 +178,8 @@ def extract(args, config, basepath, workspace):
178 178
179 srctree = os.path.abspath(args.srctree) 179 srctree = os.path.abspath(args.srctree)
180 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, rd) 180 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, rd)
181 logger.info('Source tree extracted to %s' % srctree)
182
181 if initial_rev: 183 if initial_rev:
182 return 0 184 return 0
183 else: 185 else:
@@ -351,7 +353,6 @@ def _extract_source(srctree, keep_temp, devbranch, d):
351 bb.process.run('git checkout patches', cwd=srcsubdir) 353 bb.process.run('git checkout patches', cwd=srcsubdir)
352 354
353 shutil.move(srcsubdir, srctree) 355 shutil.move(srcsubdir, srctree)
354 logger.info('Source tree extracted to %s' % srctree)
355 finally: 356 finally:
356 bb.logger.setLevel(origlevel) 357 bb.logger.setLevel(origlevel)
357 358
@@ -451,6 +452,7 @@ def modify(args, config, basepath, workspace):
451 initial_rev = _extract_source(args.srctree, False, args.branch, rd) 452 initial_rev = _extract_source(args.srctree, False, args.branch, rd)
452 if not initial_rev: 453 if not initial_rev:
453 return 1 454 return 1
455 logger.info('Source tree extracted to %s' % srctree)
454 # Get list of commits since this revision 456 # Get list of commits since this revision
455 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=args.srctree) 457 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=args.srctree)
456 commits = stdout.split() 458 commits = stdout.split()
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
new file mode 100644
index 0000000000..86443b0735
--- /dev/null
+++ b/scripts/lib/devtool/upgrade.py
@@ -0,0 +1,354 @@
1# Development tool - upgrade command plugin
2#
3# Copyright (C) 2014-2015 Intel Corporation
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17#
18"""Devtool upgrade plugin"""
19
20import os
21import sys
22import re
23import shutil
24import tempfile
25import logging
26import argparse
27import scriptutils
28import errno
29import bb
30import oe.recipeutils
31from devtool import standard
32from devtool import exec_build_env_command, setup_tinfoil, DevtoolError, parse_recipe
33
34logger = logging.getLogger('devtool')
35
36def plugin_init(pluginlist):
37 """Plugin initialization"""
38 pass
39
40def _run(cmd, cwd=''):
41 logger.debug("Running command %s> %s" % (cwd,cmd))
42 return bb.process.run('%s' % cmd, cwd=cwd)
43
44def _get_srctree(tmpdir):
45 srctree = tmpdir
46 dirs = os.listdir(tmpdir)
47 if len(dirs) == 1:
48 srctree = os.path.join(tmpdir, dirs[0])
49 return srctree
50
51def _copy_source_code(orig, dest):
52 for path in standard._ls_tree(orig):
53 dest_dir = os.path.join(dest, os.path.dirname(path))
54 bb.utils.mkdirhier(dest_dir)
55 dest_path = os.path.join(dest, path)
56 os.rename(os.path.join(orig, path), dest_path)
57
58def _get_checksums(rf):
59 import re
60 checksums = {}
61 with open(rf) as f:
62 for line in f:
63 for cs in ['md5sum', 'sha256sum']:
64 m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line)
65 if m:
66 checksums[cs] = m.group(1)
67 return checksums
68
69def _replace_checksums(rf, md5, sha256):
70 if not md5 and not sha256:
71 return
72 checksums = {'md5sum':md5, 'sha256sum':sha256}
73 with open(rf + ".tmp", "w+") as tmprf:
74 with open(rf) as f:
75 for line in f:
76 m = None
77 for cs in checksums.keys():
78 m = re.match("^SRC_URI\[%s\].*=.*\"(.*)\"" % cs, line)
79 if m:
80 if checksums[cs]:
81 oldcheck = m.group(1)
82 newcheck = checksums[cs]
83 line = line.replace(oldcheck, newcheck)
84 break
85 tmprf.write(line)
86 os.rename(rf + ".tmp", rf)
87
88
89def _remove_patch_dirs(recipefolder):
90 for root, dirs, files in os.walk(recipefolder):
91 for d in dirs:
92 shutil.rmtree(os.path.join(root,d))
93
94def _recipe_contains(rf, var):
95 import re
96 found = False
97 with open(rf) as f:
98 for line in f:
99 if re.match("^%s.*=.*" % var, line):
100 found = True
101 break
102 return found
103
104def _rename_recipe_dirs(oldpv, newpv, path):
105 for root, dirs, files in os.walk(path):
106 for olddir in dirs:
107 if olddir.find(oldpv) != -1:
108 newdir = olddir.replace(oldpv, newpv)
109 if olddir != newdir:
110 _run('mv %s %s' % (olddir, newdir))
111
112def _rename_recipe_file(bpn, oldpv, newpv, path):
113 oldrecipe = "%s_%s.bb" % (bpn, oldpv)
114 newrecipe = "%s_%s.bb" % (bpn, newpv)
115 if os.path.isfile(os.path.join(path, oldrecipe)):
116 if oldrecipe != newrecipe:
117 _run('mv %s %s' % (oldrecipe, newrecipe), cwd=path)
118 else:
119 recipe = "%s_git.bb" % bpn
120 if os.path.isfile(os.path.join(path, recipe)):
121 newrecipe = recipe
122 raise DevtoolError("Original recipe not found on workspace")
123 return os.path.join(path, newrecipe)
124
125def _rename_recipe_files(bpn, oldpv, newpv, path):
126 _rename_recipe_dirs(oldpv, newpv, path)
127 return _rename_recipe_file(bpn, oldpv, newpv, path)
128
129def _use_external_build(same_dir, no_same_dir, d):
130 b_is_s = True
131 if no_same_dir:
132 logger.info('using separate build directory since --no-same-dir specified')
133 b_is_s = False
134 elif same_dir:
135 logger.info('using source tree as build directory since --same-dir specified')
136 elif bb.data.inherits_class('autotools-brokensep', d):
137 logger.info('using source tree as build directory since original recipe inherits autotools-brokensep')
138 elif d.getVar('B', True) == os.path.abspath(d.getVar('S', True)):
139 logger.info('using source tree as build directory since that is the default for this recipe')
140 else:
141 b_is_s = False
142 return b_is_s
143
144def _write_append(rc, srctree, same_dir, no_same_dir, rev, workspace, d):
145 """Writes an append file"""
146 if not os.path.exists(rc):
147 raise DevtoolError("bbappend not created because %s does not exist" % rc)
148
149 appendpath = os.path.join(workspace, 'appends')
150 if not os.path.exists(appendpath):
151 bb.utils.mkdirhier(appendpath)
152
153 brf = os.path.basename(os.path.splitext(rc)[0]) # rc basename
154
155 srctree = os.path.abspath(srctree)
156 pn = d.getVar('PN',True)
157 af = os.path.join(appendpath, '%s.bbappend' % brf)
158 with open(af, 'w') as f:
159 f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
160 f.write('inherit externalsrc\n')
161 f.write(('# NOTE: We use pn- overrides here to avoid affecting'
162 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n'))
163 f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree))
164 if _use_external_build(same_dir, no_same_dir, d):
165 f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree))
166 if rev:
167 f.write('\n# initial_rev: %s\n' % rev)
168 return af
169
170def _cleanup_on_error(rf, srctree):
171 rfp = os.path.split(rf)[0] # recipe folder
172 rfpp = os.path.split(rfp)[0] # recipes folder
173 if os.path.exists(rfp):
174 shutil.rmtree(b)
175 if not len(os.listdir(rfpp)):
176 os.rmdir(rfpp)
177 srctree = os.path.abspath(srctree)
178 if os.path.exists(srctree):
179 shutil.rmtree(srctree)
180
181def _upgrade_error(e, rf, srctree):
182 if rf:
183 cleanup_on_error(rf, srctree)
184 logger.error(e)
185 raise DevtoolError(e)
186
187def _get_uri(rd):
188 srcuris = rd.getVar('SRC_URI', True).split()
189 if not len(srcuris):
190 raise DevtoolError('SRC_URI not found on recipe')
191 srcuri = srcuris[0] # it is assumed, URI is at first position
192 srcrev = '${AUTOREV}'
193 if '://' in srcuri:
194 # Fetch a URL
195 rev_re = re.compile(';rev=([^;]+)')
196 res = rev_re.search(srcuri)
197 if res:
198 srcrev = res.group(1)
199 srcuri = rev_re.sub('', srcuri)
200 return srcuri, srcrev
201
202def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tinfoil, rd):
203 """Extract sources of a recipe with a new version"""
204
205 def __run(cmd):
206 """Simple wrapper which calls _run with srctree as cwd"""
207 return _run(cmd, srctree)
208
209 crd = rd.createCopy()
210
211 pv = crd.getVar('PV', True)
212 crd.setVar('PV', newpv)
213
214 tmpsrctree = None
215 uri, rev = _get_uri(crd)
216 if srcrev:
217 rev = srcrev
218 if uri.startswith('git://'):
219 __run('git checkout %s' % rev)
220 __run('git tag -f devtool-base-new')
221 md5 = None
222 sha256 = None
223 else:
224 __run('git checkout -b devtool-%s' % newpv)
225
226 tmpdir = tempfile.mkdtemp(prefix='devtool')
227 try:
228 md5, sha256 = scriptutils.fetch_uri(tinfoil.config_data, uri, tmpdir, rev)
229 except bb.fetch2.FetchError as e:
230 raise DevtoolError(e)
231
232 tmpsrctree = _get_srctree(tmpdir)
233
234 scrtree = os.path.abspath(srctree)
235
236 _copy_source_code(tmpsrctree, srctree)
237
238 (stdout,_) = __run('git ls-files --modified --others --exclude-standard')
239 for f in stdout.splitlines():
240 __run('git add "%s"' % f)
241
242 __run('git commit -q -m "Commit of upstream changes at version %s" --allow-empty' % newpv)
243 __run('git tag -f devtool-base-%s' % newpv)
244
245 (stdout, _) = __run('git rev-parse HEAD')
246 rev = stdout.rstrip()
247
248 if no_patch:
249 patches = oe.recipeutils.get_recipe_patches(crd)
250 if len(patches):
251 logger.warn('By user choice, the following patches will NOT be applied')
252 for patch in patches:
253 logger.warn("%s" % os.path.basename(patch))
254 else:
255 try:
256 __run('git checkout devtool-patched -b %s' % branch)
257 __run('git rebase %s' % rev)
258 if uri.startswith('git://'):
259 suffix = 'new'
260 else:
261 suffix = newpv
262 __run('git tag -f devtool-patched-%s' % suffix)
263 except bb.process.ExecutionError as e:
264 logger.warn('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
265
266 if tmpsrctree:
267 if keep_temp:
268 logger.info('Preserving temporary directory %s' % tmpsrctree)
269 else:
270 shutil.rmtree(tmpsrctree)
271
272 return (rev, md5, sha256)
273
274def _create_new_recipe(newpv, md5, sha256, workspace, rd):
275 """Creates the new recipe under workspace"""
276 crd = rd.createCopy()
277
278 bpn = crd.getVar('BPN', True)
279 path = os.path.join(workspace, 'recipes', bpn)
280 bb.utils.mkdirhier(path)
281 oe.recipeutils.copy_recipe_files(crd, path)
282
283 oldpv = crd.getVar('PV', True)
284 if not newpv:
285 newpv = oldpv
286 fullpath = _rename_recipe_files(bpn, oldpv, newpv, path)
287
288 if _recipe_contains(fullpath, 'PV') and newpv != oldpv:
289 oe.recipeutils.patch_recipe(d, fullpath, {'PV':newpv})
290
291 if md5 and sha256:
292 # Unfortunately, oe.recipeutils.patch_recipe cannot update flags.
293 # once the latter feature is implemented, we should call patch_recipe
294 # instead of the following function
295 _replace_checksums(fullpath, md5, sha256)
296
297 return fullpath
298
299def upgrade(args, config, basepath, workspace):
300 """Entry point for the devtool 'upgrade' subcommand"""
301
302 if args.recipename in workspace:
303 raise DevtoolError("recipe %s is already in your workspace" % args.recipename)
304 if not args.version and not args.srcrev:
305 raise DevtoolError("You must provide a version using the --version/-V option, or for recipes that fetch from an SCM such as git, the --srcrev/-S option")
306
307 reason = oe.recipeutils.validate_pn(args.recipename)
308 if reason:
309 raise DevtoolError(reason)
310
311 tinfoil = setup_tinfoil()
312
313 rd = parse_recipe(config, tinfoil, args.recipename, True)
314 if not rd:
315 return 1
316
317 standard._check_compatible_recipe(args.recipename, rd)
318 if rd.getVar('PV', True) == args.version and rd.getVar('SRCREV', True) == args.srcrev:
319 raise DevtoolError("Current and upgrade versions are the same version" % version)
320
321 rf = None
322 try:
323 rev1 = standard._extract_source(args.srctree, False, 'devtool-orig', rd)
324 rev2, md5, sha256 = _extract_new_source(args.version, args.srctree, args.no_patch,
325 args.srcrev, args.branch, args.keep_temp,
326 tinfoil, rd)
327 rf = _create_new_recipe(args.version, md5, sha256, config.workspace_path, rd)
328 except bb.process.CmdError as e:
329 _upgrade_error(e, rf, args.srctree)
330 except DevtoolError as e:
331 _upgrade_error(e, rf, args.srctree)
332 standard._add_md5(config, args.recipename, os.path.dirname(rf))
333
334 af = _write_append(rf, args.srctree, args.same_dir, args.no_same_dir, rev2,
335 config.workspace_path, rd)
336 standard._add_md5(config, args.recipename, af)
337 logger.info('Upgraded source extracted to %s' % args.srctree)
338 return 0
339
340def register_commands(subparsers, context):
341 """Register devtool subcommands from this plugin"""
342 parser_upgrade = subparsers.add_parser('upgrade', help='Upgrade an existing recipe',
343 description='Upgrades an existing recipe to a new upstream version')
344 parser_upgrade.add_argument('recipename', help='Name for recipe to extract the source for')
345 parser_upgrade.add_argument('srctree', help='Path to where to extract the source tree')
346 parser_upgrade.add_argument('--version', '-V', help='Version to upgrade to (PV)')
347 parser_upgrade.add_argument('--srcrev', '-S', help='Source revision to upgrade to (if fetching from an SCM such as git)')
348 parser_upgrade.add_argument('--branch', '-b', default="devtool", help='Name for new development branch to checkout (default "%(default)s")')
349 parser_upgrade.add_argument('--no-patch', action="store_true", help='Do not apply patches from the recipe to the new source code')
350 group = parser_upgrade.add_mutually_exclusive_group()
351 group.add_argument('--same-dir', '-s', help='Build in same directory as source', action="store_true")
352 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
353 parser_upgrade.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
354 parser_upgrade.set_defaults(func=upgrade)