summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2014-12-19 11:41:55 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-12-23 10:18:16 +0000
commitcd5ca4a11daff2089ba3031103b9b9dab5b3c94e (patch)
tree2c80f6f2c4a07ce9d88b47c17ca7a424665e2283
parentb7d53f2ebb3219e205ab6b368f7f41b57a5cf89a (diff)
downloadpoky-cd5ca4a11daff2089ba3031103b9b9dab5b3c94e.tar.gz
scripts/devtool: add development helper tool
Provides an easy means to work on developing applications and system components with the build system. For example to "modify" the source for an existing recipe: $ devtool modify -x pango /home/projects/pango Parsing recipes..done. NOTE: Fetching pango... NOTE: Unpacking... NOTE: Patching... NOTE: Source tree extracted to /home/projects/pango NOTE: Recipe pango now set up to build from /home/paul/projects/pango The pango source is now extracted to /home/paul/projects/pango, managed in git, with each patch as a commit, and a bbappend is created in the workspace layer to use the source in /home/paul/projects/pango when building. Additionally, you can add a new piece of software: $ devtool add pv /home/projects/pv NOTE: Recipe /path/to/workspace/recipes/pv/pv.bb has been automatically created; further editing may be required to make it fully functional The latter uses recipetool to create a skeleton recipe and again sets up a bbappend to use the source in /home/projects/pv when building. Having done a "devtool modify", can also write any changes to the external git repository back as patches next to the recipe: $ devtool update-recipe mdadm Parsing recipes..done. NOTE: Removing patch mdadm-3.2.2_fix_for_x32.patch NOTE: Removing patch gcc-4.9.patch NOTE: Updating recipe mdadm_3.3.1.bb [YOCTO #6561] [YOCTO #6653] [YOCTO #6656] (From OE-Core rev: 716d9b1f304a12bab61b15e3ce526977c055f074) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rwxr-xr-xscripts/devtool255
-rw-r--r--scripts/lib/devtool/__init__.py78
-rw-r--r--scripts/lib/devtool/standard.py545
3 files changed, 878 insertions, 0 deletions
diff --git a/scripts/devtool b/scripts/devtool
new file mode 100755
index 0000000000..d6e1b9710d
--- /dev/null
+++ b/scripts/devtool
@@ -0,0 +1,255 @@
1#!/usr/bin/env python
2
3# OpenEmbedded Development tool
4#
5# Copyright (C) 2014 Intel Corporation
6#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License version 2 as
9# published by the Free Software Foundation.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along
17# with this program; if not, write to the Free Software Foundation, Inc.,
18# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20import sys
21import os
22import argparse
23import glob
24import re
25import ConfigParser
26import subprocess
27import logging
28
29basepath = ''
30workspace = {}
31config = None
32context = None
33
34
35scripts_path = os.path.dirname(os.path.realpath(__file__))
36lib_path = scripts_path + '/lib'
37sys.path = sys.path + [lib_path]
38import scriptutils
39logger = scriptutils.logger_create('devtool')
40
41plugins = []
42
43
44class ConfigHandler(object):
45 config_file = ''
46 config_obj = None
47 init_path = ''
48 workspace_path = ''
49
50 def __init__(self, filename):
51 self.config_file = filename
52 self.config_obj = ConfigParser.SafeConfigParser()
53
54 def get(self, section, option, default=None):
55 try:
56 ret = self.config_obj.get(section, option)
57 except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
58 if default != None:
59 ret = default
60 else:
61 raise
62 return ret
63
64 def read(self):
65 if os.path.exists(self.config_file):
66 self.config_obj.read(self.config_file)
67
68 if self.config_obj.has_option('General', 'init_path'):
69 pth = self.get('General', 'init_path')
70 self.init_path = os.path.join(basepath, pth)
71 if not os.path.exists(self.init_path):
72 logger.error('init_path %s specified in config file cannot be found' % pth)
73 return False
74 else:
75 self.config_obj.add_section('General')
76
77 self.workspace_path = self.get('General', 'workspace_path', os.path.join(basepath, 'workspace'))
78 return True
79
80
81 def write(self):
82 logger.debug('writing to config file %s' % self.config_file)
83 self.config_obj.set('General', 'workspace_path', self.workspace_path)
84 with open(self.config_file, 'w') as f:
85 self.config_obj.write(f)
86
87class Context:
88 def __init__(self, **kwargs):
89 self.__dict__.update(kwargs)
90
91
92def read_workspace():
93 global workspace
94 workspace = {}
95 if not os.path.exists(os.path.join(config.workspace_path, 'conf', 'layer.conf')):
96 if context.fixed_setup:
97 logger.error("workspace layer not set up")
98 sys.exit(1)
99 else:
100 logger.info('Creating workspace layer in %s' % config.workspace_path)
101 _create_workspace(config.workspace_path, config, basepath)
102
103 logger.debug('Reading workspace in %s' % config.workspace_path)
104 externalsrc_re = re.compile(r'^EXTERNALSRC(_pn-[a-zA-Z0-9-]*)? =.*$')
105 for fn in glob.glob(os.path.join(config.workspace_path, 'appends', '*.bbappend')):
106 pn = os.path.splitext(os.path.basename(fn))[0].split('_')[0]
107 with open(fn, 'r') as f:
108 for line in f:
109 if externalsrc_re.match(line.rstrip()):
110 splitval = line.split('=', 2)
111 workspace[pn] = splitval[1].strip('" \n\r\t')
112 break
113
114def create_workspace(args, config, basepath, workspace):
115 if args.directory:
116 workspacedir = os.path.abspath(args.directory)
117 else:
118 workspacedir = os.path.abspath(os.path.join(basepath, 'workspace'))
119 _create_workspace(workspacedir, config, basepath, args.create_only)
120
121def _create_workspace(workspacedir, config, basepath, create_only=False):
122 import bb
123
124 confdir = os.path.join(workspacedir, 'conf')
125 if os.path.exists(os.path.join(confdir, 'layer.conf')):
126 logger.info('Specified workspace already set up, leaving as-is')
127 else:
128 # Add a config file
129 bb.utils.mkdirhier(confdir)
130 with open(os.path.join(confdir, 'layer.conf'), 'w') as f:
131 f.write('# ### workspace layer auto-generated by devtool ###\n')
132 f.write('BBPATH =. "$' + '{LAYERDIR}:"\n')
133 f.write('BBFILES += "$' + '{LAYERDIR}/recipes/*/*.bb \\\n')
134 f.write(' $' + '{LAYERDIR}/appends/*.bbappend"\n')
135 f.write('BBFILE_COLLECTIONS += "workspacelayer"\n')
136 f.write('BBFILE_PATTERN_workspacelayer = "^$' + '{LAYERDIR}/"\n')
137 f.write('BBFILE_PATTERN_IGNORE_EMPTY_workspacelayer = "1"\n')
138 f.write('BBFILE_PRIORITY_workspacelayer = "99"\n')
139 # Add a README file
140 with open(os.path.join(workspacedir, 'README'), 'w') as f:
141 f.write('This layer was created by the OpenEmbedded devtool utility in order to\n')
142 f.write('contain recipes and bbappends. In most instances you should use the\n')
143 f.write('devtool utility to manage files within it rather than modifying files\n')
144 f.write('directly (although recipes added with "devtool add" will often need\n')
145 f.write('direct modification.)\n')
146 f.write('\nIf you no longer need to use devtool you can remove the path to this\n')
147 f.write('workspace layer from your conf/bblayers.conf file (and then delete the\n')
148 f.write('layer, if you wish).\n')
149 if not create_only:
150 # Add the workspace layer to bblayers.conf
151 bblayers_conf = os.path.join(basepath, 'conf', 'bblayers.conf')
152 if not os.path.exists(bblayers_conf):
153 logger.error('Unable to find bblayers.conf')
154 return -1
155 bb.utils.edit_bblayers_conf(bblayers_conf, workspacedir, config.workspace_path)
156 if config.workspace_path != workspacedir:
157 # Update our config to point to the new location
158 config.workspace_path = workspacedir
159 config.write()
160
161
162def main():
163 global basepath
164 global config
165 global context
166
167 context = Context(fixed_setup=False)
168
169 # Default basepath
170 basepath = os.path.dirname(os.path.abspath(__file__))
171 pth = basepath
172 while pth != '' and pth != os.sep:
173 if os.path.exists(os.path.join(pth, '.devtoolbase')):
174 context.fixed_setup = True
175 basepath = pth
176 break
177 pth = os.path.dirname(pth)
178
179 parser = argparse.ArgumentParser(description="OpenEmbedded development tool",
180 epilog="Use %(prog)s <command> --help to get help on a specific command")
181 parser.add_argument('--basepath', help='Base directory of SDK / build directory')
182 parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
183 parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
184 parser.add_argument('--color', help='Colorize output', choices=['auto', 'always', 'never'], default='auto')
185
186 subparsers = parser.add_subparsers(dest="subparser_name")
187
188 if not context.fixed_setup:
189 parser_create_workspace = subparsers.add_parser('create-workspace', help='Set up a workspace')
190 parser_create_workspace.add_argument('directory', nargs='?', help='Directory for the workspace')
191 parser_create_workspace.add_argument('--create-only', action="store_true", help='Only create the workspace, do not alter configuration')
192 parser_create_workspace.set_defaults(func=create_workspace)
193
194 scriptutils.load_plugins(logger, plugins, os.path.join(scripts_path, 'lib', 'devtool'))
195 for plugin in plugins:
196 if hasattr(plugin, 'register_commands'):
197 plugin.register_commands(subparsers, context)
198
199 args = parser.parse_args()
200
201 if args.debug:
202 logger.setLevel(logging.DEBUG)
203 elif args.quiet:
204 logger.setLevel(logging.ERROR)
205
206 if args.basepath:
207 # Override
208 basepath = args.basepath
209 elif not context.fixed_setup:
210 basepath = os.environ.get('BUILDDIR')
211 if not basepath:
212 logger.error("This script can only be run after initialising the build environment (e.g. by using oe-init-build-env)")
213 sys.exit(1)
214
215 logger.debug('Using basepath %s' % basepath)
216
217 config = ConfigHandler(os.path.join(basepath, 'conf', 'devtool.conf'))
218 if not config.read():
219 return -1
220
221 bitbake_subdir = config.get('General', 'bitbake_subdir', '')
222 if bitbake_subdir:
223 # Normally set for use within the SDK
224 logger.debug('Using bitbake subdir %s' % bitbake_subdir)
225 sys.path.insert(0, os.path.join(basepath, bitbake_subdir, 'lib'))
226 core_meta_subdir = config.get('General', 'core_meta_subdir')
227 sys.path.insert(0, os.path.join(basepath, core_meta_subdir, 'lib'))
228 else:
229 # Standard location
230 import scriptpath
231 bitbakepath = scriptpath.add_bitbake_lib_path()
232 if not bitbakepath:
233 logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
234 sys.exit(1)
235 logger.debug('Using standard bitbake path %s' % bitbakepath)
236 scriptpath.add_oe_lib_path()
237
238 scriptutils.logger_setup_color(logger, args.color)
239
240 if args.subparser_name != 'create-workspace':
241 read_workspace()
242
243 ret = args.func(args, config, basepath, workspace)
244
245 return ret
246
247
248if __name__ == "__main__":
249 try:
250 ret = main()
251 except Exception:
252 ret = 1
253 import traceback
254 traceback.print_exc(5)
255 sys.exit(ret)
diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py
new file mode 100644
index 0000000000..3f8158e24a
--- /dev/null
+++ b/scripts/lib/devtool/__init__.py
@@ -0,0 +1,78 @@
1#!/usr/bin/env python
2
3# Development tool - utility functions for plugins
4#
5# Copyright (C) 2014 Intel Corporation
6#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License version 2 as
9# published by the Free Software Foundation.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along
17# with this program; if not, write to the Free Software Foundation, Inc.,
18# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20
21import os
22import sys
23import subprocess
24import logging
25
26logger = logging.getLogger('devtool')
27
28def exec_build_env_command(init_path, builddir, cmd, watch=False, **options):
29 import bb
30 if not 'cwd' in options:
31 options["cwd"] = builddir
32 if init_path:
33 logger.debug('Executing command: "%s" using init path %s' % (cmd, init_path))
34 init_prefix = '. %s %s > /dev/null && ' % (init_path, builddir)
35 else:
36 logger.debug('Executing command "%s"' % cmd)
37 init_prefix = ''
38 if watch:
39 if sys.stdout.isatty():
40 # Fool bitbake into thinking it's outputting to a terminal (because it is, indirectly)
41 cmd = 'script -q -c "%s" /dev/null' % cmd
42 return exec_watch('%s%s' % (init_prefix, cmd), **options)
43 else:
44 return bb.process.run('%s%s' % (init_prefix, cmd), **options)
45
46def exec_watch(cmd, **options):
47 if isinstance(cmd, basestring) and not "shell" in options:
48 options["shell"] = True
49
50 process = subprocess.Popen(
51 cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **options
52 )
53
54 buf = ''
55 while True:
56 out = process.stdout.read(1)
57 if out:
58 sys.stdout.write(out)
59 sys.stdout.flush()
60 buf += out
61 elif out == '' and process.poll() != None:
62 break
63 return buf
64
65def setup_tinfoil():
66 import scriptpath
67 bitbakepath = scriptpath.add_bitbake_lib_path()
68 if not bitbakepath:
69 logger.error("Unable to find bitbake by searching parent directory of this script or PATH")
70 sys.exit(1)
71
72 import bb.tinfoil
73 import logging
74 tinfoil = bb.tinfoil.Tinfoil()
75 tinfoil.prepare(False)
76 tinfoil.logger.setLevel(logging.WARNING)
77 return tinfoil
78
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
new file mode 100644
index 0000000000..69bb228487
--- /dev/null
+++ b/scripts/lib/devtool/standard.py
@@ -0,0 +1,545 @@
1# Development tool - standard commands plugin
2#
3# Copyright (C) 2014 Intel Corporation
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
18import os
19import sys
20import re
21import shutil
22import glob
23import tempfile
24import logging
25import argparse
26from devtool import exec_build_env_command, setup_tinfoil
27
28logger = logging.getLogger('devtool')
29
30def plugin_init(pluginlist):
31 pass
32
33
34def add(args, config, basepath, workspace):
35 import bb
36 import oe.recipeutils
37
38 if args.recipename in workspace:
39 logger.error("recipe %s is already in your workspace" % args.recipename)
40 return -1
41
42 reason = oe.recipeutils.validate_pn(args.recipename)
43 if reason:
44 logger.error(reason)
45 return -1
46
47 srctree = os.path.abspath(args.srctree)
48 appendpath = os.path.join(config.workspace_path, 'appends')
49 if not os.path.exists(appendpath):
50 os.makedirs(appendpath)
51
52 recipedir = os.path.join(config.workspace_path, 'recipes', args.recipename)
53 bb.utils.mkdirhier(recipedir)
54 if args.version:
55 if '_' in args.version or ' ' in args.version:
56 logger.error('Invalid version string "%s"' % args.version)
57 return -1
58 bp = "%s_%s" % (args.recipename, args.version)
59 else:
60 bp = args.recipename
61 recipefile = os.path.join(recipedir, "%s.bb" % bp)
62 if sys.stdout.isatty():
63 color = 'always'
64 else:
65 color = args.color
66 stdout, stderr = exec_build_env_command(config.init_path, basepath, 'recipetool --color=%s create -o %s %s' % (color, recipefile, srctree))
67 logger.info('Recipe %s has been automatically created; further editing may be required to make it fully functional' % recipefile)
68
69 _add_md5(config, args.recipename, recipefile)
70
71 initial_rev = None
72 if os.path.exists(os.path.join(srctree, '.git')):
73 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
74 initial_rev = stdout.rstrip()
75
76 appendfile = os.path.join(appendpath, '%s.bbappend' % args.recipename)
77 with open(appendfile, 'w') as f:
78 f.write('inherit externalsrc\n')
79 f.write('EXTERNALSRC = "%s"\n' % srctree)
80 if initial_rev:
81 f.write('\n# initial_rev: %s\n' % initial_rev)
82
83 _add_md5(config, args.recipename, appendfile)
84
85 return 0
86
87
88def _get_recipe_file(cooker, pn):
89 import oe.recipeutils
90 recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
91 if not recipefile:
92 skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
93 if skipreasons:
94 logger.error('\n'.join(skipreasons))
95 else:
96 logger.error("Unable to find any recipe file matching %s" % pn)
97 return recipefile
98
99
100def extract(args, config, basepath, workspace):
101 import bb
102 import oe.recipeutils
103
104 tinfoil = setup_tinfoil()
105
106 recipefile = _get_recipe_file(tinfoil.cooker, args.recipename)
107 if not recipefile:
108 # Error already logged
109 return -1
110 rd = oe.recipeutils.parse_recipe(recipefile, tinfoil.config_data)
111
112 srctree = os.path.abspath(args.srctree)
113 initial_rev = _extract_source(srctree, args.keep_temp, args.branch, rd)
114 if initial_rev:
115 return 0
116 else:
117 return -1
118
119
120def _extract_source(srctree, keep_temp, devbranch, d):
121 import bb.event
122
123 def eventfilter(name, handler, event, d):
124 if name == 'base_eventhandler':
125 return True
126 else:
127 return False
128
129 if hasattr(bb.event, 'set_eventfilter'):
130 bb.event.set_eventfilter(eventfilter)
131
132 pn = d.getVar('PN', True)
133
134 if pn == 'perf':
135 logger.error("The perf recipe does not actually check out source and thus cannot be supported by this tool")
136 return None
137
138 if 'work-shared' in d.getVar('S', True):
139 logger.error("The %s recipe uses a shared workdir which this tool does not currently support" % pn)
140 return None
141
142 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC', True):
143 logger.error("externalsrc is currently enabled for the %s recipe. This prevents the normal do_patch task from working. You will need to disable this first." % pn)
144 return None
145
146 if os.path.exists(srctree):
147 if not os.path.isdir(srctree):
148 logger.error("output path %s exists and is not a directory" % srctree)
149 return None
150 elif os.listdir(srctree):
151 logger.error("output path %s already exists and is non-empty" % srctree)
152 return None
153
154 # Prepare for shutil.move later on
155 bb.utils.mkdirhier(srctree)
156 os.rmdir(srctree)
157
158 initial_rev = None
159 tempdir = tempfile.mkdtemp(prefix='devtool')
160 try:
161 crd = d.createCopy()
162 # Make a subdir so we guard against WORKDIR==S
163 workdir = os.path.join(tempdir, 'workdir')
164 crd.setVar('WORKDIR', workdir)
165 crd.setVar('T', os.path.join(tempdir, 'temp'))
166
167 # FIXME: This is very awkward. Unfortunately it's not currently easy to properly
168 # execute tasks outside of bitbake itself, until then this has to suffice if we
169 # are to handle e.g. linux-yocto's extra tasks
170 executed = []
171 def exec_task_func(func, report):
172 if not func in executed:
173 deps = crd.getVarFlag(func, 'deps')
174 if deps:
175 for taskdepfunc in deps:
176 exec_task_func(taskdepfunc, True)
177 if report:
178 logger.info('Executing %s...' % func)
179 fn = d.getVar('FILE', True)
180 localdata = bb.build._task_data(fn, func, crd)
181 bb.build.exec_func(func, localdata)
182 executed.append(func)
183
184 logger.info('Fetching %s...' % pn)
185 exec_task_func('do_fetch', False)
186 logger.info('Unpacking...')
187 exec_task_func('do_unpack', False)
188 srcsubdir = crd.getVar('S', True)
189 if srcsubdir != workdir and os.path.dirname(srcsubdir) != workdir:
190 # Handle if S is set to a subdirectory of the source
191 srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
192
193 patchdir = os.path.join(srcsubdir, 'patches')
194 haspatches = False
195 if os.path.exists(patchdir):
196 if os.listdir(patchdir):
197 haspatches = True
198 else:
199 os.rmdir(patchdir)
200
201 if not bb.data.inherits_class('kernel-yocto', d):
202 if not os.listdir(srcsubdir):
203 logger.error("no source unpacked to S, perhaps the %s recipe doesn't use any source?" % pn)
204 return None
205
206 if not os.path.exists(os.path.join(srcsubdir, '.git')):
207 bb.process.run('git init', cwd=srcsubdir)
208 bb.process.run('git add .', cwd=srcsubdir)
209 bb.process.run('git commit -q -m "Initial commit from upstream at version %s"' % crd.getVar('PV', True), cwd=srcsubdir)
210
211 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
212 initial_rev = stdout.rstrip()
213
214 bb.process.run('git checkout -b %s' % devbranch, cwd=srcsubdir)
215 bb.process.run('git tag -f devtool-base', cwd=srcsubdir)
216
217 crd.setVar('PATCHTOOL', 'git')
218
219 logger.info('Patching...')
220 exec_task_func('do_patch', False)
221
222 bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
223
224 if os.path.exists(patchdir):
225 shutil.rmtree(patchdir)
226 if haspatches:
227 bb.process.run('git checkout patches', cwd=srcsubdir)
228
229 shutil.move(srcsubdir, srctree)
230 logger.info('Source tree extracted to %s' % srctree)
231 finally:
232 if keep_temp:
233 logger.info('Preserving temporary directory %s' % tempdir)
234 else:
235 shutil.rmtree(tempdir)
236 return initial_rev
237
238def _add_md5(config, recipename, filename):
239 import bb.utils
240 md5 = bb.utils.md5_file(filename)
241 with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a') as f:
242 f.write('%s|%s|%s\n' % (recipename, os.path.relpath(filename, config.workspace_path), md5))
243
244def _check_preserve(config, recipename):
245 import bb.utils
246 origfile = os.path.join(config.workspace_path, '.devtool_md5')
247 newfile = os.path.join(config.workspace_path, '.devtool_md5_new')
248 preservepath = os.path.join(config.workspace_path, 'attic')
249 with open(origfile, 'r') as f:
250 with open(newfile, 'w') as tf:
251 for line in f.readlines():
252 splitline = line.rstrip().split('|')
253 if splitline[0] == recipename:
254 removefile = os.path.join(config.workspace_path, splitline[1])
255 md5 = bb.utils.md5_file(removefile)
256 if splitline[2] != md5:
257 bb.utils.mkdirhier(preservepath)
258 preservefile = os.path.basename(removefile)
259 logger.warn('File %s modified since it was written, preserving in %s' % (preservefile, preservepath))
260 shutil.move(removefile, os.path.join(preservepath, preservefile))
261 else:
262 os.remove(removefile)
263 else:
264 tf.write(line)
265 os.rename(newfile, origfile)
266
267 return False
268
269
270def modify(args, config, basepath, workspace):
271 import bb
272 import oe.recipeutils
273
274 if args.recipename in workspace:
275 logger.error("recipe %s is already in your workspace" % args.recipename)
276 return -1
277
278 if not args.extract:
279 if not os.path.isdir(args.srctree):
280 logger.error("directory %s does not exist or not a directory (specify -x to extract source from recipe)" % args.srctree)
281 return -1
282
283 tinfoil = setup_tinfoil()
284
285 recipefile = _get_recipe_file(tinfoil.cooker, args.recipename)
286 if not recipefile:
287 # Error already logged
288 return -1
289 rd = oe.recipeutils.parse_recipe(recipefile, tinfoil.config_data)
290
291 initial_rev = None
292 commits = []
293 srctree = os.path.abspath(args.srctree)
294 if args.extract:
295 initial_rev = _extract_source(args.srctree, False, args.branch, rd)
296 if not initial_rev:
297 return -1
298 # Get list of commits since this revision
299 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=args.srctree)
300 commits = stdout.split()
301 else:
302 if os.path.exists(os.path.join(args.srctree, '.git')):
303 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=args.srctree)
304 initial_rev = stdout.rstrip()
305
306 # Handle if S is set to a subdirectory of the source
307 s = rd.getVar('S', True)
308 workdir = rd.getVar('WORKDIR', True)
309 if s != workdir and os.path.dirname(s) != workdir:
310 srcsubdir = os.sep.join(os.path.relpath(s, workdir).split(os.sep)[1:])
311 srctree = os.path.join(srctree, srcsubdir)
312
313 appendpath = os.path.join(config.workspace_path, 'appends')
314 if not os.path.exists(appendpath):
315 os.makedirs(appendpath)
316
317 appendname = os.path.splitext(os.path.basename(recipefile))[0]
318 if args.wildcard:
319 appendname = re.sub(r'_.*', '_%', appendname)
320 appendfile = os.path.join(appendpath, appendname + '.bbappend')
321 with open(appendfile, 'w') as f:
322 f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
323 f.write('inherit externalsrc\n')
324 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n')
325 f.write('EXTERNALSRC_pn-%s = "%s"\n' % (args.recipename, srctree))
326 if bb.data.inherits_class('autotools-brokensep', rd):
327 logger.info('using source tree as build directory since original recipe inherits autotools-brokensep')
328 f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (args.recipename, srctree))
329 if initial_rev:
330 f.write('\n# initial_rev: %s\n' % initial_rev)
331 for commit in commits:
332 f.write('# commit: %s\n' % commit)
333
334 _add_md5(config, args.recipename, appendfile)
335
336 logger.info('Recipe %s now set up to build from %s' % (args.recipename, srctree))
337
338 return 0
339
340
341def update_recipe(args, config, basepath, workspace):
342 if not args.recipename in workspace:
343 logger.error("no recipe named %s in your workspace" % args.recipename)
344 return -1
345
346 # Get initial revision from bbappend
347 appends = glob.glob(os.path.join(config.workspace_path, 'appends', '%s_*.bbappend' % args.recipename))
348 if not appends:
349 logger.error('unable to find workspace bbappend for recipe %s' % args.recipename)
350 return -1
351
352 tinfoil = setup_tinfoil()
353 import bb
354 from oe.patch import GitApplyTree
355 import oe.recipeutils
356
357 srctree = workspace[args.recipename]
358 commits = []
359 update_rev = None
360 if args.initial_rev:
361 initial_rev = args.initial_rev
362 else:
363 initial_rev = None
364 with open(appends[0], 'r') as f:
365 for line in f:
366 if line.startswith('# initial_rev:'):
367 initial_rev = line.split(':')[-1].strip()
368 elif line.startswith('# commit:'):
369 commits.append(line.split(':')[-1].strip())
370
371 if initial_rev:
372 # Find first actually changed revision
373 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree)
374 newcommits = stdout.split()
375 for i in xrange(min(len(commits), len(newcommits))):
376 if newcommits[i] == commits[i]:
377 update_rev = commits[i]
378
379 if not initial_rev:
380 logger.error('Unable to find initial revision - please specify it with --initial-rev')
381 return -1
382
383 if not update_rev:
384 update_rev = initial_rev
385
386 # Find list of existing patches in recipe file
387 recipefile = _get_recipe_file(tinfoil.cooker, args.recipename)
388 if not recipefile:
389 # Error already logged
390 return -1
391 rd = oe.recipeutils.parse_recipe(recipefile, tinfoil.config_data)
392 existing_patches = oe.recipeutils.get_recipe_patches(rd)
393
394 removepatches = []
395 if not args.no_remove:
396 # Get all patches from source tree and check if any should be removed
397 tempdir = tempfile.mkdtemp(prefix='devtool')
398 try:
399 GitApplyTree.extractPatches(srctree, initial_rev, tempdir)
400 newpatches = os.listdir(tempdir)
401 for patch in existing_patches:
402 patchfile = os.path.basename(patch)
403 if patchfile not in newpatches:
404 removepatches.append(patch)
405 finally:
406 shutil.rmtree(tempdir)
407
408 # Get updated patches from source tree
409 tempdir = tempfile.mkdtemp(prefix='devtool')
410 try:
411 GitApplyTree.extractPatches(srctree, update_rev, tempdir)
412
413 # Match up and replace existing patches with corresponding new patches
414 updatepatches = False
415 updaterecipe = False
416 newpatches = os.listdir(tempdir)
417 for patch in existing_patches:
418 patchfile = os.path.basename(patch)
419 if patchfile in newpatches:
420 logger.info('Updating patch %s' % patchfile)
421 shutil.move(os.path.join(tempdir, patchfile), patch)
422 newpatches.remove(patchfile)
423 updatepatches = True
424 srcuri = (rd.getVar('SRC_URI', False) or '').split()
425 if newpatches:
426 # Add any patches left over
427 patchdir = os.path.join(os.path.dirname(recipefile), rd.getVar('BPN', True))
428 bb.utils.mkdirhier(patchdir)
429 for patchfile in newpatches:
430 logger.info('Adding new patch %s' % patchfile)
431 shutil.move(os.path.join(tempdir, patchfile), os.path.join(patchdir, patchfile))
432 srcuri.append('file://%s' % patchfile)
433 updaterecipe = True
434 if removepatches:
435 # Remove any patches that we don't need
436 for patch in removepatches:
437 patchfile = os.path.basename(patch)
438 for i in xrange(len(srcuri)):
439 if srcuri[i].startswith('file://') and os.path.basename(srcuri[i]).split(';')[0] == patchfile:
440 logger.info('Removing patch %s' % patchfile)
441 srcuri.pop(i)
442 # FIXME "git rm" here would be nice if the file in question is tracked
443 # FIXME there's a chance that this file is referred to by another recipe, in which case deleting wouldn't be the right thing to do
444 os.remove(patch)
445 updaterecipe = True
446 break
447 if updaterecipe:
448 logger.info('Updating recipe %s' % os.path.basename(recipefile))
449 oe.recipeutils.patch_recipe(rd, recipefile, {'SRC_URI': ' '.join(srcuri)})
450 elif not updatepatches:
451 # Neither patches nor recipe were updated
452 logger.info('No patches need updating')
453 finally:
454 shutil.rmtree(tempdir)
455
456 return 0
457
458
459def status(args, config, basepath, workspace):
460 if workspace:
461 for recipe, value in workspace.iteritems():
462 print("%s: %s" % (recipe, value))
463 else:
464 logger.info('No recipes currently in your workspace - you can use "devtool modify" to work on an existing recipe or "devtool add" to add a new one')
465 return 0
466
467
468def reset(args, config, basepath, workspace):
469 import bb.utils
470 if not args.recipename in workspace:
471 logger.error("no recipe named %s in your workspace" % args.recipename)
472 return -1
473 _check_preserve(config, args.recipename)
474
475 preservepath = os.path.join(config.workspace_path, 'attic', args.recipename)
476 def preservedir(origdir):
477 if os.path.exists(origdir):
478 for fn in os.listdir(origdir):
479 logger.warn('Preserving %s in %s' % (fn, preservepath))
480 bb.utils.mkdirhier(preservepath)
481 shutil.move(os.path.join(origdir, fn), os.path.join(preservepath, fn))
482 os.rmdir(origdir)
483
484 preservedir(os.path.join(config.workspace_path, 'recipes', args.recipename))
485 # We don't automatically create this dir next to appends, but the user can
486 preservedir(os.path.join(config.workspace_path, 'appends', args.recipename))
487 return 0
488
489
490def build(args, config, basepath, workspace):
491 import bb
492 if not args.recipename in workspace:
493 logger.error("no recipe named %s in your workspace" % args.recipename)
494 return -1
495 exec_build_env_command(config.init_path, basepath, 'bitbake -c install %s' % args.recipename, watch=True)
496
497 return 0
498
499
500def register_commands(subparsers, context):
501 parser_add = subparsers.add_parser('add', help='Add a new recipe',
502 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
503 parser_add.add_argument('recipename', help='Name for new recipe to add')
504 parser_add.add_argument('srctree', help='Path to external source tree')
505 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
506 parser_add.set_defaults(func=add)
507
508 parser_add = subparsers.add_parser('modify', help='Modify the source for an existing recipe',
509 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
510 parser_add.add_argument('recipename', help='Name for recipe to edit')
511 parser_add.add_argument('srctree', help='Path to external source tree')
512 parser_add.add_argument('--wildcard', '-w', action="store_true", help='Use wildcard for unversioned bbappend')
513 parser_add.add_argument('--extract', '-x', action="store_true", help='Extract source as well')
514 parser_add.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
515 parser_add.set_defaults(func=modify)
516
517 parser_add = subparsers.add_parser('extract', help='Extract the source for an existing recipe',
518 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
519 parser_add.add_argument('recipename', help='Name for recipe to extract the source for')
520 parser_add.add_argument('srctree', help='Path to where to extract the source tree')
521 parser_add.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout')
522 parser_add.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
523 parser_add.set_defaults(func=extract)
524
525 parser_add = subparsers.add_parser('update-recipe', help='Apply changes from external source tree to recipe',
526 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
527 parser_add.add_argument('recipename', help='Name of recipe to update')
528 parser_add.add_argument('--initial-rev', help='Starting revision for patches')
529 parser_add.add_argument('--no-remove', '-n', action="store_true", help='Don\'t remove patches, only add or update')
530 parser_add.set_defaults(func=update_recipe)
531
532 parser_status = subparsers.add_parser('status', help='Show status',
533 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
534 parser_status.set_defaults(func=status)
535
536 parser_build = subparsers.add_parser('build', help='Build recipe',
537 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
538 parser_build.add_argument('recipename', help='Recipe to build')
539 parser_build.set_defaults(func=build)
540
541 parser_reset = subparsers.add_parser('reset', help='Remove a recipe from your workspace',
542 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
543 parser_reset.add_argument('recipename', help='Recipe to reset')
544 parser_reset.set_defaults(func=reset)
545