summaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2016-03-02 23:44:21 +1300
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-03-07 00:11:38 +0000
commit25d9c4e02a90b1fd8c6a2036d29fd2cf87eca098 (patch)
tree29242ee8c4d7a9a12d7da5f40080f17d511a15ec /meta
parent41eb36dc4ce7feece2f63c7cae80478f5376459e (diff)
downloadpoky-25d9c4e02a90b1fd8c6a2036d29fd2cf87eca098.tar.gz
devtool: add build-sdk subcommand
Add a build-sdk command which is only available within the extensible SDK that builds a derivative extensible SDK. The idea is recipes in the workspace become a part of the new SDK - for example, this allows taking a vendor provided SDK, adding a few libs and then producing a new SDK with those included. When normally building the extensible SDK, the workspace is excluded; here we need to copy into the new SDK (renaming it in the process); the recipes' task signatures become locked and thus the sources are no longer needed, so they are removed along with the workspace bbappends which would interfere with the locked signatures. Additionally we need to just copy the configuration files (i.e. local.conf and auto.conf) rather than filtering and appending to them since that work has already been done when constructing the original SDK. The extra sstate artifacts from workspace recipes are also determined and copied into the new SDK in minimal mode (on the assumption that you won't set up a new sstate mirror). This reuses some code from build-image, so that needed to be generalised to allow that. Implements [YOCTO #8892]. (From OE-Core rev: 59e207ff6dd4b50a8905e14bc9292cf2794f4e7a) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
-rw-r--r--meta/classes/populate_sdk_ext.bbclass148
-rw-r--r--meta/lib/oe/copy_buildsystem.py59
2 files changed, 142 insertions, 65 deletions
diff --git a/meta/classes/populate_sdk_ext.bbclass b/meta/classes/populate_sdk_ext.bbclass
index 6079166980..6afc53d284 100644
--- a/meta/classes/populate_sdk_ext.bbclass
+++ b/meta/classes/populate_sdk_ext.bbclass
@@ -89,7 +89,14 @@ python copy_buildsystem () {
89 # Copy in all metadata layers + bitbake (as repositories) 89 # Copy in all metadata layers + bitbake (as repositories)
90 buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d) 90 buildsystem = oe.copy_buildsystem.BuildSystem('extensible SDK', d)
91 baseoutpath = d.getVar('SDK_OUTPUT', True) + '/' + d.getVar('SDKPATH', True) 91 baseoutpath = d.getVar('SDK_OUTPUT', True) + '/' + d.getVar('SDKPATH', True)
92 layers_copied = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers') 92
93 # Determine if we're building a derivative extensible SDK (from devtool build-sdk)
94 derivative = (d.getVar('SDK_DERIVATIVE', True) or '') == '1'
95 if derivative:
96 workspace_name = 'orig-workspace'
97 else:
98 workspace_name = None
99 layers_copied = buildsystem.copy_bitbake_and_layers(baseoutpath + '/layers', workspace_name)
93 100
94 sdkbblayers = [] 101 sdkbblayers = []
95 corebase = os.path.basename(d.getVar('COREBASE', True)) 102 corebase = os.path.basename(d.getVar('COREBASE', True))
@@ -158,75 +165,81 @@ python copy_buildsystem () {
158 f.write(' "\n') 165 f.write(' "\n')
159 166
160 # Create local.conf 167 # Create local.conf
161 local_conf_whitelist = (d.getVar('SDK_LOCAL_CONF_WHITELIST', True) or '').split()
162 local_conf_blacklist = (d.getVar('SDK_LOCAL_CONF_BLACKLIST', True) or '').split()
163 def handle_var(varname, origvalue, op, newlines):
164 if varname in local_conf_blacklist or (origvalue.strip().startswith('/') and not varname in local_conf_whitelist):
165 newlines.append('# Removed original setting of %s\n' % varname)
166 return None, op, 0, True
167 else:
168 return origvalue, op, 0, True
169 varlist = ['[^#=+ ]*']
170 builddir = d.getVar('TOPDIR', True) 168 builddir = d.getVar('TOPDIR', True)
171 with open(builddir + '/conf/local.conf', 'r') as f: 169 if derivative:
172 oldlines = f.readlines() 170 shutil.copyfile(builddir + '/conf/local.conf', baseoutpath + '/conf/local.conf')
173 (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var) 171 else:
172 local_conf_whitelist = (d.getVar('SDK_LOCAL_CONF_WHITELIST', True) or '').split()
173 local_conf_blacklist = (d.getVar('SDK_LOCAL_CONF_BLACKLIST', True) or '').split()
174 def handle_var(varname, origvalue, op, newlines):
175 if varname in local_conf_blacklist or (origvalue.strip().startswith('/') and not varname in local_conf_whitelist):
176 newlines.append('# Removed original setting of %s\n' % varname)
177 return None, op, 0, True
178 else:
179 return origvalue, op, 0, True
180 varlist = ['[^#=+ ]*']
181 with open(builddir + '/conf/local.conf', 'r') as f:
182 oldlines = f.readlines()
183 (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var)
174 184
175 with open(baseoutpath + '/conf/local.conf', 'w') as f: 185 with open(baseoutpath + '/conf/local.conf', 'w') as f:
176 f.write('# WARNING: this configuration has been automatically generated and in\n') 186 f.write('# WARNING: this configuration has been automatically generated and in\n')
177 f.write('# most cases should not be edited. If you need more flexibility than\n') 187 f.write('# most cases should not be edited. If you need more flexibility than\n')
178 f.write('# this configuration provides, it is strongly suggested that you set\n') 188 f.write('# this configuration provides, it is strongly suggested that you set\n')
179 f.write('# up a proper instance of the full build system and use that instead.\n\n') 189 f.write('# up a proper instance of the full build system and use that instead.\n\n')
180 for line in newlines: 190 for line in newlines:
181 if line.strip() and not line.startswith('#'): 191 if line.strip() and not line.startswith('#'):
182 f.write(line) 192 f.write(line)
183 # Write a newline just in case there's none at the end of the original 193 # Write a newline just in case there's none at the end of the original
184 f.write('\n') 194 f.write('\n')
185 195
186 f.write('INHERIT += "%s"\n\n' % 'uninative') 196 f.write('INHERIT += "%s"\n\n' % 'uninative')
187 f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False)) 197 f.write('CONF_VERSION = "%s"\n\n' % d.getVar('CONF_VERSION', False))
188 198
189 # Some classes are not suitable for SDK, remove them from INHERIT 199 # Some classes are not suitable for SDK, remove them from INHERIT
190 f.write('INHERIT_remove = "%s"\n' % d.getVar('SDK_INHERIT_BLACKLIST', False)) 200 f.write('INHERIT_remove = "%s"\n' % d.getVar('SDK_INHERIT_BLACKLIST', False))
191 201
192 # Bypass the default connectivity check if any 202 # Bypass the default connectivity check if any
193 f.write('CONNECTIVITY_CHECK_URIS = ""\n\n') 203 f.write('CONNECTIVITY_CHECK_URIS = ""\n\n')
194 204
195 # Ensure locked sstate cache objects are re-used without error 205 # Ensure locked sstate cache objects are re-used without error
196 f.write('SIGGEN_LOCKEDSIGS_CHECK_LEVEL = "none"\n\n') 206 f.write('SIGGEN_LOCKEDSIGS_CHECK_LEVEL = "none"\n\n')
197 207
198 # Hide the config information from bitbake output (since it's fixed within the SDK) 208 # Hide the config information from bitbake output (since it's fixed within the SDK)
199 f.write('BUILDCFG_HEADER = ""\n') 209 f.write('BUILDCFG_HEADER = ""\n')
200 210
201 # Allow additional config through sdk-extra.conf 211 # Allow additional config through sdk-extra.conf
202 fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d) 212 fn = bb.cookerdata.findConfigFile('sdk-extra.conf', d)
203 if fn: 213 if fn:
204 with open(fn, 'r') as xf: 214 with open(fn, 'r') as xf:
205 for line in xf: 215 for line in xf:
206 f.write(line) 216 f.write(line)
207 217
208 # If you define a sdk_extraconf() function then it can contain additional config 218 # If you define a sdk_extraconf() function then it can contain additional config
209 # (Though this is awkward; sdk-extra.conf should probably be used instead) 219 # (Though this is awkward; sdk-extra.conf should probably be used instead)
210 extraconf = (d.getVar('sdk_extraconf', True) or '').strip() 220 extraconf = (d.getVar('sdk_extraconf', True) or '').strip()
211 if extraconf: 221 if extraconf:
212 # Strip off any leading / trailing spaces 222 # Strip off any leading / trailing spaces
213 for line in extraconf.splitlines(): 223 for line in extraconf.splitlines():
214 f.write(line.strip() + '\n') 224 f.write(line.strip() + '\n')
215 225
216 f.write('require conf/locked-sigs.inc\n') 226 f.write('require conf/locked-sigs.inc\n')
217 227
218 if os.path.exists(builddir + '/conf/auto.conf'): 228 if os.path.exists(builddir + '/conf/auto.conf'):
219 with open(builddir + '/conf/auto.conf', 'r') as f: 229 if derivative:
220 oldlines = f.readlines() 230 shutil.copyfile(builddir + '/conf/auto.conf', baseoutpath + '/conf/auto.conf')
221 (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var) 231 else:
222 with open(baseoutpath + '/conf/auto.conf', 'w') as f: 232 with open(builddir + '/conf/auto.conf', 'r') as f:
223 f.write('# WARNING: this configuration has been automatically generated and in\n') 233 oldlines = f.readlines()
224 f.write('# most cases should not be edited. If you need more flexibility than\n') 234 (updated, newlines) = bb.utils.edit_metadata(oldlines, varlist, handle_var)
225 f.write('# this configuration provides, it is strongly suggested that you set\n') 235 with open(baseoutpath + '/conf/auto.conf', 'w') as f:
226 f.write('# up a proper instance of the full build system and use that instead.\n\n') 236 f.write('# WARNING: this configuration has been automatically generated and in\n')
227 for line in newlines: 237 f.write('# most cases should not be edited. If you need more flexibility than\n')
228 if line.strip() and not line.startswith('#'): 238 f.write('# this configuration provides, it is strongly suggested that you set\n')
229 f.write(line) 239 f.write('# up a proper instance of the full build system and use that instead.\n\n')
240 for line in newlines:
241 if line.strip() and not line.startswith('#'):
242 f.write(line)
230 243
231 # Filter the locked signatures file to just the sstate tasks we are interested in 244 # Filter the locked signatures file to just the sstate tasks we are interested in
232 excluded_targets = d.getVar('SDK_TARGETS', True) 245 excluded_targets = d.getVar('SDK_TARGETS', True)
@@ -253,7 +266,24 @@ python copy_buildsystem () {
253 lockedsigs_pruned, 266 lockedsigs_pruned,
254 lockedsigs_copy) 267 lockedsigs_copy)
255 268
256 if d.getVar('SDK_EXT_TYPE', True) != 'minimal': 269 if d.getVar('SDK_EXT_TYPE', True) == 'minimal':
270 if derivative:
271 # Assume the user is not going to set up an additional sstate
272 # mirror, thus we need to copy the additional artifacts (from
273 # workspace recipes) into the derivative SDK
274 lockedsigs_orig = d.getVar('TOPDIR', True) + '/conf/locked-sigs.inc'
275 if os.path.exists(lockedsigs_orig):
276 lockedsigs_extra = d.getVar('WORKDIR', True) + '/locked-sigs-extra.inc'
277 oe.copy_buildsystem.merge_lockedsigs(None,
278 lockedsigs_orig,
279 lockedsigs_pruned,
280 None,
281 lockedsigs_extra)
282 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_extra,
283 d.getVar('SSTATE_DIR', True),
284 sstate_out, d,
285 fixedlsbstring)
286 else:
257 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned, 287 oe.copy_buildsystem.create_locked_sstate_cache(lockedsigs_pruned,
258 d.getVar('SSTATE_DIR', True), 288 d.getVar('SSTATE_DIR', True),
259 sstate_out, d, 289 sstate_out, d,
diff --git a/meta/lib/oe/copy_buildsystem.py b/meta/lib/oe/copy_buildsystem.py
index 5074a43cb9..dd0e6641e3 100644
--- a/meta/lib/oe/copy_buildsystem.py
+++ b/meta/lib/oe/copy_buildsystem.py
@@ -20,7 +20,7 @@ class BuildSystem(object):
20 self.layerdirs = d.getVar('BBLAYERS', True).split() 20 self.layerdirs = d.getVar('BBLAYERS', True).split()
21 self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE', True) or "").split() 21 self.layers_exclude = (d.getVar('SDK_LAYERS_EXCLUDE', True) or "").split()
22 22
23 def copy_bitbake_and_layers(self, destdir): 23 def copy_bitbake_and_layers(self, destdir, workspace_name=None):
24 # Copy in all metadata layers + bitbake (as repositories) 24 # Copy in all metadata layers + bitbake (as repositories)
25 layers_copied = [] 25 layers_copied = []
26 bb.utils.mkdirhier(destdir) 26 bb.utils.mkdirhier(destdir)
@@ -34,6 +34,14 @@ class BuildSystem(object):
34 if layer_exclude in layers: 34 if layer_exclude in layers:
35 layers.remove(layer_exclude) 35 layers.remove(layer_exclude)
36 36
37 workspace_newname = workspace_name
38 if workspace_newname:
39 layernames = [os.path.basename(layer) for layer in layers]
40 extranum = 0
41 while workspace_newname in layernames:
42 extranum += 1
43 workspace_newname = '%s-%d' % (workspace_name, extranum)
44
37 corebase_files = self.d.getVar('COREBASE_FILES', True).split() 45 corebase_files = self.d.getVar('COREBASE_FILES', True).split()
38 corebase_files = [corebase + '/' +x for x in corebase_files] 46 corebase_files = [corebase + '/' +x for x in corebase_files]
39 # Make sure bitbake goes in 47 # Make sure bitbake goes in
@@ -42,18 +50,24 @@ class BuildSystem(object):
42 50
43 for layer in layers: 51 for layer in layers:
44 layerconf = os.path.join(layer, 'conf', 'layer.conf') 52 layerconf = os.path.join(layer, 'conf', 'layer.conf')
53 layernewname = os.path.basename(layer)
54 workspace = False
45 if os.path.exists(layerconf): 55 if os.path.exists(layerconf):
46 with open(layerconf, 'r') as f: 56 with open(layerconf, 'r') as f:
47 if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"): 57 if f.readline().startswith("# ### workspace layer auto-generated by devtool ###"):
48 bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context)) 58 if workspace_newname:
49 continue 59 layernewname = workspace_newname
60 workspace = True
61 else:
62 bb.plain("NOTE: Excluding local workspace layer %s from %s" % (layer, self.context))
63 continue
50 64
51 # If the layer was already under corebase, leave it there 65 # If the layer was already under corebase, leave it there
52 # since layers such as meta have issues when moved. 66 # since layers such as meta have issues when moved.
53 layerdestpath = destdir 67 layerdestpath = destdir
54 if corebase == os.path.dirname(layer): 68 if corebase == os.path.dirname(layer):
55 layerdestpath += '/' + os.path.basename(corebase) 69 layerdestpath += '/' + os.path.basename(corebase)
56 layerdestpath += '/' + os.path.basename(layer) 70 layerdestpath += '/' + layernewname
57 71
58 layer_relative = os.path.relpath(layerdestpath, 72 layer_relative = os.path.relpath(layerdestpath,
59 destdir) 73 destdir)
@@ -73,6 +87,38 @@ class BuildSystem(object):
73 else: 87 else:
74 _smart_copy(layer, layerdestpath) 88 _smart_copy(layer, layerdestpath)
75 89
90 if workspace:
91 # Make some adjustments original workspace layer
92 # Drop sources (recipe tasks will be locked, so we don't need them)
93 srcdir = os.path.join(layerdestpath, 'sources')
94 if os.path.isdir(srcdir):
95 shutil.rmtree(srcdir)
96 # Drop all bbappends except the one for the image the SDK is being built for
97 # (because of externalsrc, the workspace bbappends will interfere with the
98 # locked signatures if present, and we don't need them anyway)
99 image_bbappend = os.path.splitext(os.path.basename(self.d.getVar('FILE', True)))[0] + '.bbappend'
100 appenddir = os.path.join(layerdestpath, 'appends')
101 if os.path.isdir(appenddir):
102 for fn in os.listdir(appenddir):
103 if fn == image_bbappend:
104 continue
105 else:
106 os.remove(os.path.join(appenddir, fn))
107 # Drop README
108 readme = os.path.join(layerdestpath, 'README')
109 if os.path.exists(readme):
110 os.remove(readme)
111 # Filter out comments in layer.conf and change layer name
112 layerconf = os.path.join(layerdestpath, 'conf', 'layer.conf')
113 with open(layerconf, 'r') as f:
114 origlines = f.readlines()
115 with open(layerconf, 'w') as f:
116 for line in origlines:
117 if line.startswith('#'):
118 continue
119 line = line.replace('workspacelayer', workspace_newname)
120 f.write(line)
121
76 return layers_copied 122 return layers_copied
77 123
78def generate_locked_sigs(sigfile, d): 124def generate_locked_sigs(sigfile, d):
@@ -123,7 +169,7 @@ def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_outpu
123 if line.endswith('\\\n'): 169 if line.endswith('\\\n'):
124 if not line in merged[invalue]: 170 if not line in merged[invalue]:
125 target, task = line.strip().split(':')[:2] 171 target, task = line.strip().split(':')[:2]
126 if task in copy_tasks: 172 if not copy_tasks or task in copy_tasks:
127 tocopy[invalue].append(line) 173 tocopy[invalue].append(line)
128 merged[invalue].append(line) 174 merged[invalue].append(line)
129 else: 175 else:
@@ -150,7 +196,8 @@ def merge_lockedsigs(copy_tasks, lockedsigs_main, lockedsigs_extra, merged_outpu
150 f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes)) 196 f.write('SIGGEN_LOCKEDSIGS_TYPES = "%s"\n' % ' '.join(fulltypes))
151 197
152 write_sigs_file(copy_output, tocopy.keys(), tocopy) 198 write_sigs_file(copy_output, tocopy.keys(), tocopy)
153 write_sigs_file(merged_output, arch_order, merged) 199 if merged_output:
200 write_sigs_file(merged_output, arch_order, merged)
154 201
155def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring=""): 202def create_locked_sstate_cache(lockedsigs, input_sstate_cache, output_sstate_cache, d, fixedlsbstring=""):
156 bb.note('Generating sstate-cache...') 203 bb.note('Generating sstate-cache...')