summaryrefslogtreecommitdiffstats
path: root/scripts/lib
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/lib')
-rw-r--r--scripts/lib/argparse_oe.py2
-rw-r--r--scripts/lib/build_perf/report.py3
-rw-r--r--scripts/lib/buildstats.py38
-rw-r--r--scripts/lib/checklayer/__init__.py58
-rw-r--r--scripts/lib/checklayer/cases/bsp.py4
-rw-r--r--scripts/lib/checklayer/cases/common.py46
-rw-r--r--scripts/lib/checklayer/cases/distro.py2
-rw-r--r--scripts/lib/devtool/__init__.py27
-rw-r--r--scripts/lib/devtool/build_image.py2
-rw-r--r--scripts/lib/devtool/build_sdk.py2
-rw-r--r--scripts/lib/devtool/deploy.py240
-rw-r--r--scripts/lib/devtool/ide_plugins/__init__.py282
-rw-r--r--scripts/lib/devtool/ide_plugins/ide_code.py463
-rw-r--r--scripts/lib/devtool/ide_plugins/ide_none.py53
-rwxr-xr-xscripts/lib/devtool/ide_sdk.py1070
-rw-r--r--scripts/lib/devtool/menuconfig.py4
-rw-r--r--scripts/lib/devtool/sdk.py5
-rw-r--r--scripts/lib/devtool/search.py5
-rw-r--r--scripts/lib/devtool/standard.py540
-rw-r--r--scripts/lib/devtool/upgrade.py196
-rw-r--r--scripts/lib/recipetool/append.py84
-rw-r--r--scripts/lib/recipetool/create.py360
-rw-r--r--scripts/lib/recipetool/create_buildsys.py43
-rw-r--r--scripts/lib/recipetool/create_buildsys_python.py1100
-rw-r--r--scripts/lib/recipetool/create_go.py777
-rw-r--r--scripts/lib/recipetool/create_kmod.py2
-rw-r--r--scripts/lib/recipetool/create_npm.py103
-rw-r--r--scripts/lib/recipetool/licenses.csv37
-rw-r--r--scripts/lib/recipetool/setvar.py1
-rw-r--r--scripts/lib/resulttool/log.py13
-rw-r--r--scripts/lib/resulttool/regression.py281
-rw-r--r--scripts/lib/resulttool/report.py5
-rw-r--r--scripts/lib/resulttool/resultutils.py8
-rw-r--r--scripts/lib/scriptutils.py25
-rw-r--r--scripts/lib/wic/canned-wks/common.wks.inc2
-rw-r--r--scripts/lib/wic/canned-wks/directdisk-gpt.wks2
-rw-r--r--scripts/lib/wic/canned-wks/efi-bootdisk.wks.in2
-rw-r--r--scripts/lib/wic/canned-wks/mkefidisk.wks2
-rw-r--r--scripts/lib/wic/canned-wks/qemuloongarch.wks3
-rw-r--r--scripts/lib/wic/canned-wks/qemux86-directdisk.wks2
-rw-r--r--scripts/lib/wic/engine.py6
-rw-r--r--scripts/lib/wic/filemap.py7
-rw-r--r--scripts/lib/wic/help.py18
-rw-r--r--scripts/lib/wic/ksparser.py13
-rw-r--r--scripts/lib/wic/misc.py14
-rw-r--r--scripts/lib/wic/partition.py73
-rw-r--r--scripts/lib/wic/pluginbase.py8
-rw-r--r--scripts/lib/wic/plugins/imager/direct.py134
-rw-r--r--scripts/lib/wic/plugins/source/bootimg-efi.py211
-rw-r--r--scripts/lib/wic/plugins/source/bootimg-partition.py9
-rw-r--r--scripts/lib/wic/plugins/source/bootimg-pcbios.py12
-rw-r--r--scripts/lib/wic/plugins/source/empty.py59
-rw-r--r--scripts/lib/wic/plugins/source/isoimage-isohybrid.py2
-rw-r--r--scripts/lib/wic/plugins/source/rawcopy.py42
-rw-r--r--scripts/lib/wic/plugins/source/rootfs.py13
55 files changed, 5407 insertions, 1108 deletions
diff --git a/scripts/lib/argparse_oe.py b/scripts/lib/argparse_oe.py
index 94a4ac5011..176b732bbc 100644
--- a/scripts/lib/argparse_oe.py
+++ b/scripts/lib/argparse_oe.py
@@ -1,4 +1,6 @@
1# 1#
2# Copyright OpenEmbedded Contributors
3#
2# SPDX-License-Identifier: GPL-2.0-only 4# SPDX-License-Identifier: GPL-2.0-only
3# 5#
4 6
diff --git a/scripts/lib/build_perf/report.py b/scripts/lib/build_perf/report.py
index 4e8e2a8a93..ab77424cc7 100644
--- a/scripts/lib/build_perf/report.py
+++ b/scripts/lib/build_perf/report.py
@@ -4,7 +4,8 @@
4# SPDX-License-Identifier: GPL-2.0-only 4# SPDX-License-Identifier: GPL-2.0-only
5# 5#
6"""Handling of build perf test reports""" 6"""Handling of build perf test reports"""
7from collections import OrderedDict, Mapping, namedtuple 7from collections import OrderedDict, namedtuple
8from collections.abc import Mapping
8from datetime import datetime, timezone 9from datetime import datetime, timezone
9from numbers import Number 10from numbers import Number
10from statistics import mean, stdev, variance 11from statistics import mean, stdev, variance
diff --git a/scripts/lib/buildstats.py b/scripts/lib/buildstats.py
index c69b5bf4d7..6db60d5bcf 100644
--- a/scripts/lib/buildstats.py
+++ b/scripts/lib/buildstats.py
@@ -8,7 +8,7 @@ import json
8import logging 8import logging
9import os 9import os
10import re 10import re
11from collections import namedtuple,OrderedDict 11from collections import namedtuple
12from statistics import mean 12from statistics import mean
13 13
14 14
@@ -79,8 +79,8 @@ class BSTask(dict):
79 return self['rusage']['ru_oublock'] 79 return self['rusage']['ru_oublock']
80 80
81 @classmethod 81 @classmethod
82 def from_file(cls, buildstat_file): 82 def from_file(cls, buildstat_file, fallback_end=0):
83 """Read buildstat text file""" 83 """Read buildstat text file. fallback_end is an optional end time for tasks that are not recorded as finishing."""
84 bs_task = cls() 84 bs_task = cls()
85 log.debug("Reading task buildstats from %s", buildstat_file) 85 log.debug("Reading task buildstats from %s", buildstat_file)
86 end_time = None 86 end_time = None
@@ -108,7 +108,10 @@ class BSTask(dict):
108 bs_task[ru_type][ru_key] = val 108 bs_task[ru_type][ru_key] = val
109 elif key == 'Status': 109 elif key == 'Status':
110 bs_task['status'] = val 110 bs_task['status'] = val
111 if end_time is not None and start_time is not None: 111 # If the task didn't finish, fill in the fallback end time if specified
112 if start_time and not end_time and fallback_end:
113 end_time = fallback_end
114 if start_time and end_time:
112 bs_task['elapsed_time'] = end_time - start_time 115 bs_task['elapsed_time'] = end_time - start_time
113 else: 116 else:
114 raise BSError("{} looks like a invalid buildstats file".format(buildstat_file)) 117 raise BSError("{} looks like a invalid buildstats file".format(buildstat_file))
@@ -226,25 +229,44 @@ class BuildStats(dict):
226 epoch = match.group('epoch') 229 epoch = match.group('epoch')
227 return name, epoch, version, revision 230 return name, epoch, version, revision
228 231
232 @staticmethod
233 def parse_top_build_stats(path):
234 """
235 Parse the top-level build_stats file for build-wide start and duration.
236 """
237 start = elapsed = 0
238 with open(path) as fobj:
239 for line in fobj.readlines():
240 key, val = line.split(':', 1)
241 val = val.strip()
242 if key == 'Build Started':
243 start = float(val)
244 elif key == "Elapsed time":
245 elapsed = float(val.split()[0])
246 return start, elapsed
247
229 @classmethod 248 @classmethod
230 def from_dir(cls, path): 249 def from_dir(cls, path):
231 """Load buildstats from a buildstats directory""" 250 """Load buildstats from a buildstats directory"""
232 if not os.path.isfile(os.path.join(path, 'build_stats')): 251 top_stats = os.path.join(path, 'build_stats')
252 if not os.path.isfile(top_stats):
233 raise BSError("{} does not look like a buildstats directory".format(path)) 253 raise BSError("{} does not look like a buildstats directory".format(path))
234 254
235 log.debug("Reading buildstats directory %s", path) 255 log.debug("Reading buildstats directory %s", path)
236
237 buildstats = cls() 256 buildstats = cls()
257 build_started, build_elapsed = buildstats.parse_top_build_stats(top_stats)
258 build_end = build_started + build_elapsed
259
238 subdirs = os.listdir(path) 260 subdirs = os.listdir(path)
239 for dirname in subdirs: 261 for dirname in subdirs:
240 recipe_dir = os.path.join(path, dirname) 262 recipe_dir = os.path.join(path, dirname)
241 if not os.path.isdir(recipe_dir): 263 if dirname == "reduced_proc_pressure" or not os.path.isdir(recipe_dir):
242 continue 264 continue
243 name, epoch, version, revision = cls.split_nevr(dirname) 265 name, epoch, version, revision = cls.split_nevr(dirname)
244 bsrecipe = BSRecipe(name, epoch, version, revision) 266 bsrecipe = BSRecipe(name, epoch, version, revision)
245 for task in os.listdir(recipe_dir): 267 for task in os.listdir(recipe_dir):
246 bsrecipe.tasks[task] = BSTask.from_file( 268 bsrecipe.tasks[task] = BSTask.from_file(
247 os.path.join(recipe_dir, task)) 269 os.path.join(recipe_dir, task), build_end)
248 if name in buildstats: 270 if name in buildstats:
249 raise BSError("Cannot handle multiple versions of the same " 271 raise BSError("Cannot handle multiple versions of the same "
250 "package ({})".format(name)) 272 "package ({})".format(name))
diff --git a/scripts/lib/checklayer/__init__.py b/scripts/lib/checklayer/__init__.py
index fe545607bb..62ecdfe390 100644
--- a/scripts/lib/checklayer/__init__.py
+++ b/scripts/lib/checklayer/__init__.py
@@ -16,6 +16,7 @@ class LayerType(Enum):
16 BSP = 0 16 BSP = 0
17 DISTRO = 1 17 DISTRO = 1
18 SOFTWARE = 2 18 SOFTWARE = 2
19 CORE = 3
19 ERROR_NO_LAYER_CONF = 98 20 ERROR_NO_LAYER_CONF = 98
20 ERROR_BSP_DISTRO = 99 21 ERROR_BSP_DISTRO = 99
21 22
@@ -43,7 +44,7 @@ def _get_layer_collections(layer_path, lconf=None, data=None):
43 44
44 ldata.setVar('LAYERDIR', layer_path) 45 ldata.setVar('LAYERDIR', layer_path)
45 try: 46 try:
46 ldata = bb.parse.handle(lconf, ldata, include=True) 47 ldata = bb.parse.handle(lconf, ldata, include=True, baseconfig=True)
47 except: 48 except:
48 raise RuntimeError("Parsing of layer.conf from layer: %s failed" % layer_path) 49 raise RuntimeError("Parsing of layer.conf from layer: %s failed" % layer_path)
49 ldata.expandVarref('LAYERDIR') 50 ldata.expandVarref('LAYERDIR')
@@ -106,7 +107,13 @@ def _detect_layer(layer_path):
106 if distros: 107 if distros:
107 is_distro = True 108 is_distro = True
108 109
109 if is_bsp and is_distro: 110 layer['collections'] = _get_layer_collections(layer['path'])
111
112 if layer_name == "meta" and "core" in layer['collections']:
113 layer['type'] = LayerType.CORE
114 layer['conf']['machines'] = machines
115 layer['conf']['distros'] = distros
116 elif is_bsp and is_distro:
110 layer['type'] = LayerType.ERROR_BSP_DISTRO 117 layer['type'] = LayerType.ERROR_BSP_DISTRO
111 elif is_bsp: 118 elif is_bsp:
112 layer['type'] = LayerType.BSP 119 layer['type'] = LayerType.BSP
@@ -117,8 +124,6 @@ def _detect_layer(layer_path):
117 else: 124 else:
118 layer['type'] = LayerType.SOFTWARE 125 layer['type'] = LayerType.SOFTWARE
119 126
120 layer['collections'] = _get_layer_collections(layer['path'])
121
122 return layer 127 return layer
123 128
124def detect_layers(layer_directories, no_auto): 129def detect_layers(layer_directories, no_auto):
@@ -146,7 +151,7 @@ def detect_layers(layer_directories, no_auto):
146 151
147 return layers 152 return layers
148 153
149def _find_layer_depends(depend, layers): 154def _find_layer(depend, layers):
150 for layer in layers: 155 for layer in layers:
151 if 'collections' not in layer: 156 if 'collections' not in layer:
152 continue 157 continue
@@ -156,7 +161,28 @@ def _find_layer_depends(depend, layers):
156 return layer 161 return layer
157 return None 162 return None
158 163
159def add_layer_dependencies(bblayersconf, layer, layers, logger): 164def sanity_check_layers(layers, logger):
165 """
166 Check that we didn't find duplicate collection names, as the layer that will
167 be used is non-deterministic. The precise check is duplicate collections
168 with different patterns, as the same pattern being repeated won't cause
169 problems.
170 """
171 import collections
172
173 passed = True
174 seen = collections.defaultdict(set)
175 for layer in layers:
176 for name, data in layer.get("collections", {}).items():
177 seen[name].add(data["pattern"])
178
179 for name, patterns in seen.items():
180 if len(patterns) > 1:
181 passed = False
182 logger.error("Collection %s found multiple times: %s" % (name, ", ".join(patterns)))
183 return passed
184
185def get_layer_dependencies(layer, layers, logger):
160 def recurse_dependencies(depends, layer, layers, logger, ret = []): 186 def recurse_dependencies(depends, layer, layers, logger, ret = []):
161 logger.debug('Processing dependencies %s for layer %s.' % \ 187 logger.debug('Processing dependencies %s for layer %s.' % \
162 (depends, layer['name'])) 188 (depends, layer['name']))
@@ -166,7 +192,7 @@ def add_layer_dependencies(bblayersconf, layer, layers, logger):
166 if depend == 'core': 192 if depend == 'core':
167 continue 193 continue
168 194
169 layer_depend = _find_layer_depends(depend, layers) 195 layer_depend = _find_layer(depend, layers)
170 if not layer_depend: 196 if not layer_depend:
171 logger.error('Layer %s depends on %s and isn\'t found.' % \ 197 logger.error('Layer %s depends on %s and isn\'t found.' % \
172 (layer['name'], depend)) 198 (layer['name'], depend))
@@ -203,6 +229,11 @@ def add_layer_dependencies(bblayersconf, layer, layers, logger):
203 layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends) 229 layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends)
204 230
205 # Note: [] (empty) is allowed, None is not! 231 # Note: [] (empty) is allowed, None is not!
232 return layer_depends
233
234def add_layer_dependencies(bblayersconf, layer, layers, logger):
235
236 layer_depends = get_layer_dependencies(layer, layers, logger)
206 if layer_depends is None: 237 if layer_depends is None:
207 return False 238 return False
208 else: 239 else:
@@ -256,7 +287,7 @@ def check_command(error_msg, cmd, cwd=None):
256 raise RuntimeError(msg) 287 raise RuntimeError(msg)
257 return output 288 return output
258 289
259def get_signatures(builddir, failsafe=False, machine=None): 290def get_signatures(builddir, failsafe=False, machine=None, extravars=None):
260 import re 291 import re
261 292
262 # some recipes needs to be excluded like meta-world-pkgdata 293 # some recipes needs to be excluded like meta-world-pkgdata
@@ -267,13 +298,16 @@ def get_signatures(builddir, failsafe=False, machine=None):
267 sigs = {} 298 sigs = {}
268 tune2tasks = {} 299 tune2tasks = {}
269 300
270 cmd = 'BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" ' 301 cmd = 'BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" '
302 if extravars:
303 cmd += extravars
304 cmd += ' '
271 if machine: 305 if machine:
272 cmd += 'MACHINE=%s ' % machine 306 cmd += 'MACHINE=%s ' % machine
273 cmd += 'bitbake ' 307 cmd += 'bitbake '
274 if failsafe: 308 if failsafe:
275 cmd += '-k ' 309 cmd += '-k '
276 cmd += '-S none world' 310 cmd += '-S lockedsigs world'
277 sigs_file = os.path.join(builddir, 'locked-sigs.inc') 311 sigs_file = os.path.join(builddir, 'locked-sigs.inc')
278 if os.path.exists(sigs_file): 312 if os.path.exists(sigs_file):
279 os.unlink(sigs_file) 313 os.unlink(sigs_file)
@@ -290,8 +324,8 @@ def get_signatures(builddir, failsafe=False, machine=None):
290 else: 324 else:
291 raise 325 raise
292 326
293 sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$") 327 sig_regex = re.compile(r"^(?P<task>.*:.*):(?P<hash>.*) .$")
294 tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*") 328 tune_regex = re.compile(r"(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*")
295 current_tune = None 329 current_tune = None
296 with open(sigs_file, 'r') as f: 330 with open(sigs_file, 'r') as f:
297 for line in f.readlines(): 331 for line in f.readlines():
diff --git a/scripts/lib/checklayer/cases/bsp.py b/scripts/lib/checklayer/cases/bsp.py
index 7fd56f5d36..b76163fb56 100644
--- a/scripts/lib/checklayer/cases/bsp.py
+++ b/scripts/lib/checklayer/cases/bsp.py
@@ -11,7 +11,7 @@ from checklayer.case import OECheckLayerTestCase
11class BSPCheckLayer(OECheckLayerTestCase): 11class BSPCheckLayer(OECheckLayerTestCase):
12 @classmethod 12 @classmethod
13 def setUpClass(self): 13 def setUpClass(self):
14 if self.tc.layer['type'] != LayerType.BSP: 14 if self.tc.layer['type'] not in (LayerType.BSP, LayerType.CORE):
15 raise unittest.SkipTest("BSPCheckLayer: Layer %s isn't BSP one." %\ 15 raise unittest.SkipTest("BSPCheckLayer: Layer %s isn't BSP one." %\
16 self.tc.layer['name']) 16 self.tc.layer['name'])
17 17
@@ -153,7 +153,7 @@ class BSPCheckLayer(OECheckLayerTestCase):
153 # do_build can be ignored: it is know to have 153 # do_build can be ignored: it is know to have
154 # different signatures in some cases, for example in 154 # different signatures in some cases, for example in
155 # the allarch ca-certificates due to RDEPENDS=openssl. 155 # the allarch ca-certificates due to RDEPENDS=openssl.
156 # That particular dependency is whitelisted via 156 # That particular dependency is marked via
157 # SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS, but still shows up 157 # SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS, but still shows up
158 # in the sstate signature hash because filtering it 158 # in the sstate signature hash because filtering it
159 # out would be hard and running do_build multiple 159 # out would be hard and running do_build multiple
diff --git a/scripts/lib/checklayer/cases/common.py b/scripts/lib/checklayer/cases/common.py
index b82304e361..97b16f78c8 100644
--- a/scripts/lib/checklayer/cases/common.py
+++ b/scripts/lib/checklayer/cases/common.py
@@ -6,15 +6,19 @@
6import glob 6import glob
7import os 7import os
8import unittest 8import unittest
9import re
9from checklayer import get_signatures, LayerType, check_command, get_depgraph, compare_signatures 10from checklayer import get_signatures, LayerType, check_command, get_depgraph, compare_signatures
10from checklayer.case import OECheckLayerTestCase 11from checklayer.case import OECheckLayerTestCase
11 12
12class CommonCheckLayer(OECheckLayerTestCase): 13class CommonCheckLayer(OECheckLayerTestCase):
13 def test_readme(self): 14 def test_readme(self):
15 if self.tc.layer['type'] == LayerType.CORE:
16 raise unittest.SkipTest("Core layer's README is top level")
17
14 # The top-level README file may have a suffix (like README.rst or README.txt). 18 # The top-level README file may have a suffix (like README.rst or README.txt).
15 readme_files = glob.glob(os.path.join(self.tc.layer['path'], '[Rr][Ee][Aa][Dd][Mm][Ee]*')) 19 readme_files = glob.glob(os.path.join(self.tc.layer['path'], '[Rr][Ee][Aa][Dd][Mm][Ee]*'))
16 self.assertTrue(len(readme_files) > 0, 20 self.assertTrue(len(readme_files) > 0,
17 msg="Layer doesn't contains README file.") 21 msg="Layer doesn't contain a README file.")
18 22
19 # There might be more than one file matching the file pattern above 23 # There might be more than one file matching the file pattern above
20 # (for example, README.rst and README-COPYING.rst). The one with the shortest 24 # (for example, README.rst and README-COPYING.rst). The one with the shortest
@@ -26,6 +30,16 @@ class CommonCheckLayer(OECheckLayerTestCase):
26 self.assertTrue(data, 30 self.assertTrue(data,
27 msg="Layer contains a README file but it is empty.") 31 msg="Layer contains a README file but it is empty.")
28 32
33 # If a layer's README references another README, then the checks below are not valid
34 if re.search('README', data, re.IGNORECASE):
35 return
36
37 self.assertIn('maintainer', data.lower())
38 self.assertIn('patch', data.lower())
39 # Check that there is an email address in the README
40 email_regex = re.compile(r"[^@]+@[^@]+")
41 self.assertTrue(email_regex.match(data))
42
29 def test_parse(self): 43 def test_parse(self):
30 check_command('Layer %s failed to parse.' % self.tc.layer['name'], 44 check_command('Layer %s failed to parse.' % self.tc.layer['name'],
31 'bitbake -p') 45 'bitbake -p')
@@ -43,6 +57,36 @@ class CommonCheckLayer(OECheckLayerTestCase):
43 ''' 57 '''
44 get_signatures(self.td['builddir'], failsafe=False) 58 get_signatures(self.td['builddir'], failsafe=False)
45 59
60 def test_world_inherit_class(self):
61 '''
62 This also does "bitbake -S none world" along with inheriting "yocto-check-layer"
63 class, which can do additional per-recipe test cases.
64 '''
65 msg = []
66 try:
67 get_signatures(self.td['builddir'], failsafe=False, machine=None, extravars='BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS INHERIT" INHERIT="yocto-check-layer"')
68 except RuntimeError as ex:
69 msg.append(str(ex))
70 if msg:
71 msg.insert(0, 'Layer %s failed additional checks from yocto-check-layer.bbclass\nSee below log for specific recipe parsing errors:\n' % \
72 self.tc.layer['name'])
73 self.fail('\n'.join(msg))
74
75 @unittest.expectedFailure
76 def test_patches_upstream_status(self):
77 import sys
78 sys.path.append(os.path.join(sys.path[0], '../../../../meta/lib/'))
79 import oe.qa
80 patches = []
81 for dirpath, dirs, files in os.walk(self.tc.layer['path']):
82 for filename in files:
83 if filename.endswith(".patch"):
84 ppath = os.path.join(dirpath, filename)
85 if oe.qa.check_upstream_status(ppath):
86 patches.append(ppath)
87 self.assertEqual(len(patches), 0 , \
88 msg="Found following patches with malformed or missing upstream status:\n%s" % '\n'.join([str(patch) for patch in patches]))
89
46 def test_signatures(self): 90 def test_signatures(self):
47 if self.tc.layer['type'] == LayerType.SOFTWARE and \ 91 if self.tc.layer['type'] == LayerType.SOFTWARE and \
48 not self.tc.test_software_layer_signatures: 92 not self.tc.test_software_layer_signatures:
diff --git a/scripts/lib/checklayer/cases/distro.py b/scripts/lib/checklayer/cases/distro.py
index f0bee5493c..a35332451c 100644
--- a/scripts/lib/checklayer/cases/distro.py
+++ b/scripts/lib/checklayer/cases/distro.py
@@ -11,7 +11,7 @@ from checklayer.case import OECheckLayerTestCase
11class DistroCheckLayer(OECheckLayerTestCase): 11class DistroCheckLayer(OECheckLayerTestCase):
12 @classmethod 12 @classmethod
13 def setUpClass(self): 13 def setUpClass(self):
14 if self.tc.layer['type'] != LayerType.DISTRO: 14 if self.tc.layer['type'] not in (LayerType.DISTRO, LayerType.CORE):
15 raise unittest.SkipTest("DistroCheckLayer: Layer %s isn't Distro one." %\ 15 raise unittest.SkipTest("DistroCheckLayer: Layer %s isn't Distro one." %\
16 self.tc.layer['name']) 16 self.tc.layer['name'])
17 17
diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py
index 702db669de..6133c1c5b4 100644
--- a/scripts/lib/devtool/__init__.py
+++ b/scripts/lib/devtool/__init__.py
@@ -78,12 +78,15 @@ def exec_fakeroot(d, cmd, **kwargs):
78 """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions""" 78 """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions"""
79 # Grab the command and check it actually exists 79 # Grab the command and check it actually exists
80 fakerootcmd = d.getVar('FAKEROOTCMD') 80 fakerootcmd = d.getVar('FAKEROOTCMD')
81 fakerootenv = d.getVar('FAKEROOTENV')
82 exec_fakeroot_no_d(fakerootcmd, fakerootenv, cmd, kwargs)
83
84def exec_fakeroot_no_d(fakerootcmd, fakerootenv, cmd, **kwargs):
81 if not os.path.exists(fakerootcmd): 85 if not os.path.exists(fakerootcmd):
82 logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built') 86 logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built')
83 return 2 87 return 2
84 # Set up the appropriate environment 88 # Set up the appropriate environment
85 newenv = dict(os.environ) 89 newenv = dict(os.environ)
86 fakerootenv = d.getVar('FAKEROOTENV')
87 for varvalue in fakerootenv.split(): 90 for varvalue in fakerootenv.split():
88 if '=' in varvalue: 91 if '=' in varvalue:
89 splitval = varvalue.split('=', 1) 92 splitval = varvalue.split('=', 1)
@@ -233,6 +236,28 @@ def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
233 bb.process.run('git checkout -b %s' % devbranch, cwd=repodir) 236 bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
234 bb.process.run('git tag -f %s' % basetag, cwd=repodir) 237 bb.process.run('git tag -f %s' % basetag, cwd=repodir)
235 238
239 # if recipe unpacks another git repo inside S, we need to declare it as a regular git submodule now,
240 # so we will be able to tag branches on it and extract patches when doing finish/update on the recipe
241 stdout, _ = bb.process.run("git status --porcelain", cwd=repodir)
242 found = False
243 for line in stdout.splitlines():
244 if line.endswith("/"):
245 new_dir = line.split()[1]
246 for root, dirs, files in os.walk(os.path.join(repodir, new_dir)):
247 if ".git" in dirs + files:
248 (stdout, _) = bb.process.run('git remote', cwd=root)
249 remote = stdout.splitlines()[0]
250 (stdout, _) = bb.process.run('git remote get-url %s' % remote, cwd=root)
251 remote_url = stdout.splitlines()[0]
252 logger.error(os.path.relpath(os.path.join(root, ".."), root))
253 bb.process.run('git submodule add %s %s' % (remote_url, os.path.relpath(root, os.path.join(root, ".."))), cwd=os.path.join(root, ".."))
254 found = True
255 if found:
256 oe.patch.GitApplyTree.commitIgnored("Add additional submodule from SRC_URI", dir=os.path.join(root, ".."), d=d)
257 found = False
258 if os.path.exists(os.path.join(repodir, '.gitmodules')):
259 bb.process.run('git submodule foreach --recursive "git tag -f %s"' % basetag, cwd=repodir)
260
236def recipe_to_append(recipefile, config, wildcard=False): 261def recipe_to_append(recipefile, config, wildcard=False):
237 """ 262 """
238 Convert a recipe file to a bbappend file path within the workspace. 263 Convert a recipe file to a bbappend file path within the workspace.
diff --git a/scripts/lib/devtool/build_image.py b/scripts/lib/devtool/build_image.py
index 9388abbacf..980f90ddd6 100644
--- a/scripts/lib/devtool/build_image.py
+++ b/scripts/lib/devtool/build_image.py
@@ -113,7 +113,7 @@ def build_image_task(config, basepath, workspace, image, add_packages=None, task
113 with open(appendfile, 'w') as afile: 113 with open(appendfile, 'w') as afile:
114 if packages: 114 if packages:
115 # include packages from workspace recipes into the image 115 # include packages from workspace recipes into the image
116 afile.write('IMAGE_INSTALL_append = " %s"\n' % ' '.join(packages)) 116 afile.write('IMAGE_INSTALL:append = " %s"\n' % ' '.join(packages))
117 if not task: 117 if not task:
118 logger.info('Building image %s with the following ' 118 logger.info('Building image %s with the following '
119 'additional packages: %s', image, ' '.join(packages)) 119 'additional packages: %s', image, ' '.join(packages))
diff --git a/scripts/lib/devtool/build_sdk.py b/scripts/lib/devtool/build_sdk.py
index 6fe02fff2a..1cd4831d2b 100644
--- a/scripts/lib/devtool/build_sdk.py
+++ b/scripts/lib/devtool/build_sdk.py
@@ -13,7 +13,7 @@ import shutil
13import errno 13import errno
14import sys 14import sys
15import tempfile 15import tempfile
16from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError 16from devtool import DevtoolError
17from devtool import build_image 17from devtool import build_image
18 18
19logger = logging.getLogger('devtool') 19logger = logging.getLogger('devtool')
diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py
index e5af2c95ae..b5ca8f2c2f 100644
--- a/scripts/lib/devtool/deploy.py
+++ b/scripts/lib/devtool/deploy.py
@@ -16,7 +16,7 @@ import bb.utils
16import argparse_oe 16import argparse_oe
17import oe.types 17import oe.types
18 18
19from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError 19from devtool import exec_fakeroot_no_d, setup_tinfoil, check_workspace_recipe, DevtoolError
20 20
21logger = logging.getLogger('devtool') 21logger = logging.getLogger('devtool')
22 22
@@ -133,16 +133,38 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals
133 133
134 return '\n'.join(lines) 134 return '\n'.join(lines)
135 135
136
137
138def deploy(args, config, basepath, workspace): 136def deploy(args, config, basepath, workspace):
139 """Entry point for the devtool 'deploy' subcommand""" 137 """Entry point for the devtool 'deploy' subcommand"""
140 import math 138 import oe.utils
141 import oe.recipeutils
142 import oe.package
143 139
144 check_workspace_recipe(workspace, args.recipename, checksrc=False) 140 check_workspace_recipe(workspace, args.recipename, checksrc=False)
145 141
142 tinfoil = setup_tinfoil(basepath=basepath)
143 try:
144 try:
145 rd = tinfoil.parse_recipe(args.recipename)
146 except Exception as e:
147 raise DevtoolError('Exception parsing recipe %s: %s' %
148 (args.recipename, e))
149
150 srcdir = rd.getVar('D')
151 workdir = rd.getVar('WORKDIR')
152 path = rd.getVar('PATH')
153 strip_cmd = rd.getVar('STRIP')
154 libdir = rd.getVar('libdir')
155 base_libdir = rd.getVar('base_libdir')
156 max_process = oe.utils.get_bb_number_threads(rd)
157 fakerootcmd = rd.getVar('FAKEROOTCMD')
158 fakerootenv = rd.getVar('FAKEROOTENV')
159 finally:
160 tinfoil.shutdown()
161
162 return deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_process, fakerootcmd, fakerootenv, args)
163
164def deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_process, fakerootcmd, fakerootenv, args):
165 import math
166 import oe.package
167
146 try: 168 try:
147 host, destdir = args.target.split(':') 169 host, destdir = args.target.split(':')
148 except ValueError: 170 except ValueError:
@@ -152,118 +174,108 @@ def deploy(args, config, basepath, workspace):
152 if not destdir.endswith('/'): 174 if not destdir.endswith('/'):
153 destdir += '/' 175 destdir += '/'
154 176
155 tinfoil = setup_tinfoil(basepath=basepath) 177 recipe_outdir = srcdir
156 try: 178 if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
157 try: 179 raise DevtoolError('No files to deploy - have you built the %s '
158 rd = tinfoil.parse_recipe(args.recipename) 180 'recipe? If so, the install step has not installed '
159 except Exception as e: 181 'any files.' % args.recipename)
160 raise DevtoolError('Exception parsing recipe %s: %s' % 182
161 (args.recipename, e)) 183 if args.strip and not args.dry_run:
162 recipe_outdir = rd.getVar('D') 184 # Fakeroot copy to new destination
163 if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir): 185 srcdir = recipe_outdir
164 raise DevtoolError('No files to deploy - have you built the %s ' 186 recipe_outdir = os.path.join(workdir, 'devtool-deploy-target-stripped')
165 'recipe? If so, the install step has not installed ' 187 if os.path.isdir(recipe_outdir):
166 'any files.' % args.recipename) 188 exec_fakeroot_no_d(fakerootcmd, fakerootenv, "rm -rf %s" % recipe_outdir, shell=True)
167 189 exec_fakeroot_no_d(fakerootcmd, fakerootenv, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True)
168 if args.strip and not args.dry_run: 190 os.environ['PATH'] = ':'.join([os.environ['PATH'], path or ''])
169 # Fakeroot copy to new destination 191 oe.package.strip_execs(args.recipename, recipe_outdir, strip_cmd, libdir, base_libdir, max_process)
170 srcdir = recipe_outdir 192
171 recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'deploy-target-stripped') 193 filelist = []
172 if os.path.isdir(recipe_outdir): 194 inodes = set({})
173 bb.utils.remove(recipe_outdir, True) 195 ftotalsize = 0
174 exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True) 196 for root, _, files in os.walk(recipe_outdir):
175 os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or '']) 197 for fn in files:
176 oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'), 198 fstat = os.lstat(os.path.join(root, fn))
177 rd.getVar('base_libdir'), rd) 199 # Get the size in kiB (since we'll be comparing it to the output of du -k)
178 200 # MUST use lstat() here not stat() or getfilesize() since we don't want to
179 filelist = [] 201 # dereference symlinks
180 inodes = set({}) 202 if fstat.st_ino in inodes:
181 ftotalsize = 0 203 fsize = 0
182 for root, _, files in os.walk(recipe_outdir): 204 else:
183 for fn in files: 205 fsize = int(math.ceil(float(fstat.st_size)/1024))
184 fstat = os.lstat(os.path.join(root, fn)) 206 inodes.add(fstat.st_ino)
185 # Get the size in kiB (since we'll be comparing it to the output of du -k) 207 ftotalsize += fsize
186 # MUST use lstat() here not stat() or getfilesize() since we don't want to 208 # The path as it would appear on the target
187 # dereference symlinks 209 fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
188 if fstat.st_ino in inodes: 210 filelist.append((fpath, fsize))
189 fsize = 0 211
190 else: 212 if args.dry_run:
191 fsize = int(math.ceil(float(fstat.st_size)/1024)) 213 print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
192 inodes.add(fstat.st_ino) 214 for item, _ in filelist:
193 ftotalsize += fsize 215 print(' %s' % item)
194 # The path as it would appear on the target 216 return 0
195 fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
196 filelist.append((fpath, fsize))
197
198 if args.dry_run:
199 print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
200 for item, _ in filelist:
201 print(' %s' % item)
202 return 0
203
204 extraoptions = ''
205 if args.no_host_check:
206 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
207 if not args.show_status:
208 extraoptions += ' -q'
209
210 scp_sshexec = ''
211 ssh_sshexec = 'ssh'
212 if args.ssh_exec:
213 scp_sshexec = "-S %s" % args.ssh_exec
214 ssh_sshexec = args.ssh_exec
215 scp_port = ''
216 ssh_port = ''
217 if args.port:
218 scp_port = "-P %s" % args.port
219 ssh_port = "-p %s" % args.port
220
221 if args.key:
222 extraoptions += ' -i %s' % args.key
223
224 # In order to delete previously deployed files and have the manifest file on
225 # the target, we write out a shell script and then copy it to the target
226 # so we can then run it (piping tar output to it).
227 # (We cannot use scp here, because it doesn't preserve symlinks.)
228 tmpdir = tempfile.mkdtemp(prefix='devtool')
229 try:
230 tmpscript = '/tmp/devtool_deploy.sh'
231 tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
232 shellscript = _prepare_remote_script(deploy=True,
233 verbose=args.show_status,
234 nopreserve=args.no_preserve,
235 nocheckspace=args.no_check_space)
236 # Write out the script to a file
237 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
238 f.write(shellscript)
239 # Write out the file list
240 with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
241 f.write('%d\n' % ftotalsize)
242 for fpath, fsize in filelist:
243 f.write('%s %d\n' % (fpath, fsize))
244 # Copy them to the target
245 ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
246 if ret != 0:
247 raise DevtoolError('Failed to copy script to %s - rerun with -s to '
248 'get a complete error message' % args.target)
249 finally:
250 shutil.rmtree(tmpdir)
251 217
252 # Now run the script 218 extraoptions = ''
253 ret = exec_fakeroot(rd, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True) 219 if args.no_host_check:
254 if ret != 0: 220 extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
255 raise DevtoolError('Deploy failed - rerun with -s to get a complete ' 221 if not args.show_status:
256 'error message') 222 extraoptions += ' -q'
257 223
258 logger.info('Successfully deployed %s' % recipe_outdir) 224 scp_sshexec = ''
225 ssh_sshexec = 'ssh'
226 if args.ssh_exec:
227 scp_sshexec = "-S %s" % args.ssh_exec
228 ssh_sshexec = args.ssh_exec
229 scp_port = ''
230 ssh_port = ''
231 if args.port:
232 scp_port = "-P %s" % args.port
233 ssh_port = "-p %s" % args.port
234
235 if args.key:
236 extraoptions += ' -i %s' % args.key
259 237
260 files_list = [] 238 # In order to delete previously deployed files and have the manifest file on
261 for root, _, files in os.walk(recipe_outdir): 239 # the target, we write out a shell script and then copy it to the target
262 for filename in files: 240 # so we can then run it (piping tar output to it).
263 filename = os.path.relpath(os.path.join(root, filename), recipe_outdir) 241 # (We cannot use scp here, because it doesn't preserve symlinks.)
264 files_list.append(os.path.join(destdir, filename)) 242 tmpdir = tempfile.mkdtemp(prefix='devtool')
243 try:
244 tmpscript = '/tmp/devtool_deploy.sh'
245 tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
246 shellscript = _prepare_remote_script(deploy=True,
247 verbose=args.show_status,
248 nopreserve=args.no_preserve,
249 nocheckspace=args.no_check_space)
250 # Write out the script to a file
251 with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
252 f.write(shellscript)
253 # Write out the file list
254 with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
255 f.write('%d\n' % ftotalsize)
256 for fpath, fsize in filelist:
257 f.write('%s %d\n' % (fpath, fsize))
258 # Copy them to the target
259 ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
260 if ret != 0:
261 raise DevtoolError('Failed to copy script to %s - rerun with -s to '
262 'get a complete error message' % args.target)
265 finally: 263 finally:
266 tinfoil.shutdown() 264 shutil.rmtree(tmpdir)
265
266 # Now run the script
267 ret = exec_fakeroot_no_d(fakerootcmd, fakerootenv, 'tar cf - . | %s %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True)
268 if ret != 0:
269 raise DevtoolError('Deploy failed - rerun with -s to get a complete '
270 'error message')
271
272 logger.info('Successfully deployed %s' % recipe_outdir)
273
274 files_list = []
275 for root, _, files in os.walk(recipe_outdir):
276 for filename in files:
277 filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
278 files_list.append(os.path.join(destdir, filename))
267 279
268 return 0 280 return 0
269 281
diff --git a/scripts/lib/devtool/ide_plugins/__init__.py b/scripts/lib/devtool/ide_plugins/__init__.py
new file mode 100644
index 0000000000..19c2f61c5f
--- /dev/null
+++ b/scripts/lib/devtool/ide_plugins/__init__.py
@@ -0,0 +1,282 @@
1#
2# Copyright (C) 2023-2024 Siemens AG
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6"""Devtool ide-sdk IDE plugin interface definition and helper functions"""
7
8import errno
9import json
10import logging
11import os
12import stat
13from enum import Enum, auto
14from devtool import DevtoolError
15from bb.utils import mkdirhier
16
17logger = logging.getLogger('devtool')
18
19
20class BuildTool(Enum):
21 UNDEFINED = auto()
22 CMAKE = auto()
23 MESON = auto()
24
25 @property
26 def is_c_ccp(self):
27 if self is BuildTool.CMAKE:
28 return True
29 if self is BuildTool.MESON:
30 return True
31 return False
32
33
34class GdbCrossConfig:
35 """Base class defining the GDB configuration generator interface
36
37 Generate a GDB configuration for a binary on the target device.
38 Only one instance per binary is allowed. This allows to assign unique port
39 numbers for all gdbserver instances.
40 """
41 _gdbserver_port_next = 1234
42 _binaries = []
43
44 def __init__(self, image_recipe, modified_recipe, binary, gdbserver_multi=True):
45 self.image_recipe = image_recipe
46 self.modified_recipe = modified_recipe
47 self.gdb_cross = modified_recipe.gdb_cross
48 self.binary = binary
49 if binary in GdbCrossConfig._binaries:
50 raise DevtoolError(
51 "gdbserver config for binary %s is already generated" % binary)
52 GdbCrossConfig._binaries.append(binary)
53 self.script_dir = modified_recipe.ide_sdk_scripts_dir
54 self.gdbinit_dir = os.path.join(self.script_dir, 'gdbinit')
55 self.gdbserver_multi = gdbserver_multi
56 self.binary_pretty = self.binary.replace(os.sep, '-').lstrip('-')
57 self.gdbserver_port = GdbCrossConfig._gdbserver_port_next
58 GdbCrossConfig._gdbserver_port_next += 1
59 self.id_pretty = "%d_%s" % (self.gdbserver_port, self.binary_pretty)
60 # gdbserver start script
61 gdbserver_script_file = 'gdbserver_' + self.id_pretty
62 if self.gdbserver_multi:
63 gdbserver_script_file += "_m"
64 self.gdbserver_script = os.path.join(
65 self.script_dir, gdbserver_script_file)
66 # gdbinit file
67 self.gdbinit = os.path.join(
68 self.gdbinit_dir, 'gdbinit_' + self.id_pretty)
69 # gdb start script
70 self.gdb_script = os.path.join(
71 self.script_dir, 'gdb_' + self.id_pretty)
72
73 def _gen_gdbserver_start_script(self):
74 """Generate a shell command starting the gdbserver on the remote device via ssh
75
76 GDB supports two modes:
77 multi: gdbserver remains running over several debug sessions
78 once: gdbserver terminates after the debugged process terminates
79 """
80 cmd_lines = ['#!/bin/sh']
81 if self.gdbserver_multi:
82 temp_dir = "TEMP_DIR=/tmp/gdbserver_%s; " % self.id_pretty
83 gdbserver_cmd_start = temp_dir
84 gdbserver_cmd_start += "test -f \\$TEMP_DIR/pid && exit 0; "
85 gdbserver_cmd_start += "mkdir -p \\$TEMP_DIR; "
86 gdbserver_cmd_start += "%s --multi :%s > \\$TEMP_DIR/log 2>&1 & " % (
87 self.gdb_cross.gdbserver_path, self.gdbserver_port)
88 gdbserver_cmd_start += "echo \\$! > \\$TEMP_DIR/pid;"
89
90 gdbserver_cmd_stop = temp_dir
91 gdbserver_cmd_stop += "test -f \\$TEMP_DIR/pid && kill \\$(cat \\$TEMP_DIR/pid); "
92 gdbserver_cmd_stop += "rm -rf \\$TEMP_DIR; "
93
94 gdbserver_cmd_l = []
95 gdbserver_cmd_l.append('if [ "$1" = "stop" ]; then')
96 gdbserver_cmd_l.append(' shift')
97 gdbserver_cmd_l.append(" %s %s %s %s 'sh -c \"%s\"'" % (
98 self.gdb_cross.target_device.ssh_sshexec, self.gdb_cross.target_device.ssh_port, self.gdb_cross.target_device.extraoptions, self.gdb_cross.target_device.target, gdbserver_cmd_stop))
99 gdbserver_cmd_l.append('else')
100 gdbserver_cmd_l.append(" %s %s %s %s 'sh -c \"%s\"'" % (
101 self.gdb_cross.target_device.ssh_sshexec, self.gdb_cross.target_device.ssh_port, self.gdb_cross.target_device.extraoptions, self.gdb_cross.target_device.target, gdbserver_cmd_start))
102 gdbserver_cmd_l.append('fi')
103 gdbserver_cmd = os.linesep.join(gdbserver_cmd_l)
104 else:
105 gdbserver_cmd_start = "%s --once :%s %s" % (
106 self.gdb_cross.gdbserver_path, self.gdbserver_port, self.binary)
107 gdbserver_cmd = "%s %s %s %s 'sh -c \"%s\"'" % (
108 self.gdb_cross.target_device.ssh_sshexec, self.gdb_cross.target_device.ssh_port, self.gdb_cross.target_device.extraoptions, self.gdb_cross.target_device.target, gdbserver_cmd_start)
109 cmd_lines.append(gdbserver_cmd)
110 GdbCrossConfig.write_file(self.gdbserver_script, cmd_lines, True)
111
112 def _gen_gdbinit_config(self):
113 """Generate a gdbinit file for this binary and the corresponding gdbserver configuration"""
114 gdbinit_lines = ['# This file is generated by devtool ide-sdk']
115 if self.gdbserver_multi:
116 target_help = '# gdbserver --multi :%d' % self.gdbserver_port
117 remote_cmd = 'target extended-remote'
118 else:
119 target_help = '# gdbserver :%d %s' % (
120 self.gdbserver_port, self.binary)
121 remote_cmd = 'target remote'
122 gdbinit_lines.append('# On the remote target:')
123 gdbinit_lines.append(target_help)
124 gdbinit_lines.append('# On the build machine:')
125 gdbinit_lines.append('# cd ' + self.modified_recipe.real_srctree)
126 gdbinit_lines.append(
127 '# ' + self.gdb_cross.gdb + ' -ix ' + self.gdbinit)
128
129 gdbinit_lines.append('set sysroot ' + self.modified_recipe.d)
130 gdbinit_lines.append('set substitute-path "/usr/include" "' +
131 os.path.join(self.modified_recipe.recipe_sysroot, 'usr', 'include') + '"')
132 # Disable debuginfod for now, the IDE configuration uses rootfs-dbg from the image workdir.
133 gdbinit_lines.append('set debuginfod enabled off')
134 if self.image_recipe.rootfs_dbg:
135 gdbinit_lines.append(
136 'set solib-search-path "' + self.modified_recipe.solib_search_path_str(self.image_recipe) + '"')
137 # First: Search for sources of this recipe in the workspace folder
138 if self.modified_recipe.pn in self.modified_recipe.target_dbgsrc_dir:
139 gdbinit_lines.append('set substitute-path "%s" "%s"' %
140 (self.modified_recipe.target_dbgsrc_dir, self.modified_recipe.real_srctree))
141 else:
142 logger.error(
143 "TARGET_DBGSRC_DIR must contain the recipe name PN.")
144 # Second: Search for sources of other recipes in the rootfs-dbg
145 if self.modified_recipe.target_dbgsrc_dir.startswith("/usr/src/debug"):
146 gdbinit_lines.append('set substitute-path "/usr/src/debug" "%s"' % os.path.join(
147 self.image_recipe.rootfs_dbg, "usr", "src", "debug"))
148 else:
149 logger.error(
150 "TARGET_DBGSRC_DIR must start with /usr/src/debug.")
151 else:
152 logger.warning(
153 "Cannot setup debug symbols configuration for GDB. IMAGE_GEN_DEBUGFS is not enabled.")
154 gdbinit_lines.append(
155 '%s %s:%d' % (remote_cmd, self.gdb_cross.host, self.gdbserver_port))
156 gdbinit_lines.append('set remote exec-file ' + self.binary)
157 gdbinit_lines.append(
158 'run ' + os.path.join(self.modified_recipe.d, self.binary))
159
160 GdbCrossConfig.write_file(self.gdbinit, gdbinit_lines)
161
162 def _gen_gdb_start_script(self):
163 """Generate a script starting GDB with the corresponding gdbinit configuration."""
164 cmd_lines = ['#!/bin/sh']
165 cmd_lines.append('cd ' + self.modified_recipe.real_srctree)
166 cmd_lines.append(self.gdb_cross.gdb + ' -ix ' +
167 self.gdbinit + ' "$@"')
168 GdbCrossConfig.write_file(self.gdb_script, cmd_lines, True)
169
170 def initialize(self):
171 self._gen_gdbserver_start_script()
172 self._gen_gdbinit_config()
173 self._gen_gdb_start_script()
174
175 @staticmethod
176 def write_file(script_file, cmd_lines, executable=False):
177 script_dir = os.path.dirname(script_file)
178 mkdirhier(script_dir)
179 with open(script_file, 'w') as script_f:
180 script_f.write(os.linesep.join(cmd_lines))
181 script_f.write(os.linesep)
182 if executable:
183 st = os.stat(script_file)
184 os.chmod(script_file, st.st_mode | stat.S_IEXEC)
185 logger.info("Created: %s" % script_file)
186
187
188class IdeBase:
189 """Base class defining the interface for IDE plugins"""
190
191 def __init__(self):
192 self.ide_name = 'undefined'
193 self.gdb_cross_configs = []
194
195 @classmethod
196 def ide_plugin_priority(cls):
197 """Used to find the default ide handler if --ide is not passed"""
198 return 10
199
200 def setup_shared_sysroots(self, shared_env):
201 logger.warn("Shared sysroot mode is not supported for IDE %s" %
202 self.ide_name)
203
204 def setup_modified_recipe(self, args, image_recipe, modified_recipe):
205 logger.warn("Modified recipe mode is not supported for IDE %s" %
206 self.ide_name)
207
208 def initialize_gdb_cross_configs(self, image_recipe, modified_recipe, gdb_cross_config_class=GdbCrossConfig):
209 binaries = modified_recipe.find_installed_binaries()
210 for binary in binaries:
211 gdb_cross_config = gdb_cross_config_class(
212 image_recipe, modified_recipe, binary)
213 gdb_cross_config.initialize()
214 self.gdb_cross_configs.append(gdb_cross_config)
215
216 @staticmethod
217 def gen_oe_scrtips_sym_link(modified_recipe):
218 # create a sym-link from sources to the scripts directory
219 if os.path.isdir(modified_recipe.ide_sdk_scripts_dir):
220 IdeBase.symlink_force(modified_recipe.ide_sdk_scripts_dir,
221 os.path.join(modified_recipe.real_srctree, 'oe-scripts'))
222
223 @staticmethod
224 def update_json_file(json_dir, json_file, update_dict):
225 """Update a json file
226
227 By default it uses the dict.update function. If this is not sutiable
228 the update function might be passed via update_func parameter.
229 """
230 json_path = os.path.join(json_dir, json_file)
231 logger.info("Updating IDE config file: %s (%s)" %
232 (json_file, json_path))
233 if not os.path.exists(json_dir):
234 os.makedirs(json_dir)
235 try:
236 with open(json_path) as f:
237 orig_dict = json.load(f)
238 except json.decoder.JSONDecodeError:
239 logger.info(
240 "Decoding %s failed. Probably because of comments in the json file" % json_path)
241 orig_dict = {}
242 except FileNotFoundError:
243 orig_dict = {}
244 orig_dict.update(update_dict)
245 with open(json_path, 'w') as f:
246 json.dump(orig_dict, f, indent=4)
247
248 @staticmethod
249 def symlink_force(tgt, dst):
250 try:
251 os.symlink(tgt, dst)
252 except OSError as err:
253 if err.errno == errno.EEXIST:
254 if os.readlink(dst) != tgt:
255 os.remove(dst)
256 os.symlink(tgt, dst)
257 else:
258 raise err
259
260
261def get_devtool_deploy_opts(args):
262 """Filter args for devtool deploy-target args"""
263 if not args.target:
264 return None
265 devtool_deploy_opts = [args.target]
266 if args.no_host_check:
267 devtool_deploy_opts += ["-c"]
268 if args.show_status:
269 devtool_deploy_opts += ["-s"]
270 if args.no_preserve:
271 devtool_deploy_opts += ["-p"]
272 if args.no_check_space:
273 devtool_deploy_opts += ["--no-check-space"]
274 if args.ssh_exec:
275 devtool_deploy_opts += ["-e", args.ssh.exec]
276 if args.port:
277 devtool_deploy_opts += ["-P", args.port]
278 if args.key:
279 devtool_deploy_opts += ["-I", args.key]
280 if args.strip is False:
281 devtool_deploy_opts += ["--no-strip"]
282 return devtool_deploy_opts
diff --git a/scripts/lib/devtool/ide_plugins/ide_code.py b/scripts/lib/devtool/ide_plugins/ide_code.py
new file mode 100644
index 0000000000..a62b93224e
--- /dev/null
+++ b/scripts/lib/devtool/ide_plugins/ide_code.py
@@ -0,0 +1,463 @@
1#
2# Copyright (C) 2023-2024 Siemens AG
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6"""Devtool ide-sdk IDE plugin for VSCode and VSCodium"""
7
8import json
9import logging
10import os
11import shutil
12from devtool.ide_plugins import BuildTool, IdeBase, GdbCrossConfig, get_devtool_deploy_opts
13
14logger = logging.getLogger('devtool')
15
16
17class GdbCrossConfigVSCode(GdbCrossConfig):
18 def __init__(self, image_recipe, modified_recipe, binary):
19 super().__init__(image_recipe, modified_recipe, binary, False)
20
21 def initialize(self):
22 self._gen_gdbserver_start_script()
23
24
25class IdeVSCode(IdeBase):
26 """Manage IDE configurations for VSCode
27
28 Modified recipe mode:
29 - cmake: use the cmake-preset generated by devtool ide-sdk
30 - meson: meson is called via a wrapper script generated by devtool ide-sdk
31
32 Shared sysroot mode:
33 In shared sysroot mode, the cross tool-chain is exported to the user's global configuration.
34 A workspace cannot be created because there is no recipe that defines how a workspace could
35 be set up.
36 - cmake: adds a cmake-kit to .local/share/CMakeTools/cmake-tools-kits.json
37 The cmake-kit uses the environment script and the tool-chain file
38 generated by meta-ide-support.
39 - meson: Meson needs manual workspace configuration.
40 """
41
42 @classmethod
43 def ide_plugin_priority(cls):
44 """If --ide is not passed this is the default plugin"""
45 if shutil.which('code'):
46 return 100
47 return 0
48
49 def setup_shared_sysroots(self, shared_env):
50 """Expose the toolchain of the shared sysroots SDK"""
51 datadir = shared_env.ide_support.datadir
52 deploy_dir_image = shared_env.ide_support.deploy_dir_image
53 real_multimach_target_sys = shared_env.ide_support.real_multimach_target_sys
54 standalone_sysroot_native = shared_env.build_sysroots.standalone_sysroot_native
55 vscode_ws_path = os.path.join(
56 os.environ['HOME'], '.local', 'share', 'CMakeTools')
57 cmake_kits_path = os.path.join(vscode_ws_path, 'cmake-tools-kits.json')
58 oecmake_generator = "Ninja"
59 env_script = os.path.join(
60 deploy_dir_image, 'environment-setup-' + real_multimach_target_sys)
61
62 if not os.path.isdir(vscode_ws_path):
63 os.makedirs(vscode_ws_path)
64 cmake_kits_old = []
65 if os.path.exists(cmake_kits_path):
66 with open(cmake_kits_path, 'r', encoding='utf-8') as cmake_kits_file:
67 cmake_kits_old = json.load(cmake_kits_file)
68 cmake_kits = cmake_kits_old.copy()
69
70 cmake_kit_new = {
71 "name": "OE " + real_multimach_target_sys,
72 "environmentSetupScript": env_script,
73 "toolchainFile": standalone_sysroot_native + datadir + "/cmake/OEToolchainConfig.cmake",
74 "preferredGenerator": {
75 "name": oecmake_generator
76 }
77 }
78
79 def merge_kit(cmake_kits, cmake_kit_new):
80 i = 0
81 while i < len(cmake_kits):
82 if 'environmentSetupScript' in cmake_kits[i] and \
83 cmake_kits[i]['environmentSetupScript'] == cmake_kit_new['environmentSetupScript']:
84 cmake_kits[i] = cmake_kit_new
85 return
86 i += 1
87 cmake_kits.append(cmake_kit_new)
88 merge_kit(cmake_kits, cmake_kit_new)
89
90 if cmake_kits != cmake_kits_old:
91 logger.info("Updating: %s" % cmake_kits_path)
92 with open(cmake_kits_path, 'w', encoding='utf-8') as cmake_kits_file:
93 json.dump(cmake_kits, cmake_kits_file, indent=4)
94 else:
95 logger.info("Already up to date: %s" % cmake_kits_path)
96
97 cmake_native = os.path.join(
98 shared_env.build_sysroots.standalone_sysroot_native, 'usr', 'bin', 'cmake')
99 if os.path.isfile(cmake_native):
100 logger.info('cmake-kits call cmake by default. If the cmake provided by this SDK should be used, please add the following line to ".vscode/settings.json" file: "cmake.cmakePath": "%s"' % cmake_native)
101 else:
102 logger.error("Cannot find cmake native at: %s" % cmake_native)
103
104 def dot_code_dir(self, modified_recipe):
105 return os.path.join(modified_recipe.srctree, '.vscode')
106
107 def __vscode_settings_meson(self, settings_dict, modified_recipe):
108 if modified_recipe.build_tool is not BuildTool.MESON:
109 return
110 settings_dict["mesonbuild.mesonPath"] = modified_recipe.meson_wrapper
111
112 confopts = modified_recipe.mesonopts.split()
113 confopts += modified_recipe.meson_cross_file.split()
114 confopts += modified_recipe.extra_oemeson.split()
115 settings_dict["mesonbuild.configureOptions"] = confopts
116 settings_dict["mesonbuild.buildFolder"] = modified_recipe.b
117
118 def __vscode_settings_cmake(self, settings_dict, modified_recipe):
119 """Add cmake specific settings to settings.json.
120
121 Note: most settings are passed to the cmake preset.
122 """
123 if modified_recipe.build_tool is not BuildTool.CMAKE:
124 return
125 settings_dict["cmake.configureOnOpen"] = True
126 settings_dict["cmake.sourceDirectory"] = modified_recipe.real_srctree
127
128 def vscode_settings(self, modified_recipe, image_recipe):
129 files_excludes = {
130 "**/.git/**": True,
131 "**/oe-logs/**": True,
132 "**/oe-workdir/**": True,
133 "**/source-date-epoch/**": True
134 }
135 python_exclude = [
136 "**/.git/**",
137 "**/oe-logs/**",
138 "**/oe-workdir/**",
139 "**/source-date-epoch/**"
140 ]
141 files_readonly = {
142 modified_recipe.recipe_sysroot + '/**': True,
143 modified_recipe.recipe_sysroot_native + '/**': True,
144 }
145 if image_recipe.rootfs_dbg is not None:
146 files_readonly[image_recipe.rootfs_dbg + '/**'] = True
147 settings_dict = {
148 "files.watcherExclude": files_excludes,
149 "files.exclude": files_excludes,
150 "files.readonlyInclude": files_readonly,
151 "python.analysis.exclude": python_exclude
152 }
153 self.__vscode_settings_cmake(settings_dict, modified_recipe)
154 self.__vscode_settings_meson(settings_dict, modified_recipe)
155
156 settings_file = 'settings.json'
157 IdeBase.update_json_file(
158 self.dot_code_dir(modified_recipe), settings_file, settings_dict)
159
160 def __vscode_extensions_cmake(self, modified_recipe, recommendations):
161 if modified_recipe.build_tool is not BuildTool.CMAKE:
162 return
163 recommendations += [
164 "twxs.cmake",
165 "ms-vscode.cmake-tools",
166 "ms-vscode.cpptools",
167 "ms-vscode.cpptools-extension-pack",
168 "ms-vscode.cpptools-themes"
169 ]
170
171 def __vscode_extensions_meson(self, modified_recipe, recommendations):
172 if modified_recipe.build_tool is not BuildTool.MESON:
173 return
174 recommendations += [
175 'mesonbuild.mesonbuild',
176 "ms-vscode.cpptools",
177 "ms-vscode.cpptools-extension-pack",
178 "ms-vscode.cpptools-themes"
179 ]
180
181 def vscode_extensions(self, modified_recipe):
182 recommendations = []
183 self.__vscode_extensions_cmake(modified_recipe, recommendations)
184 self.__vscode_extensions_meson(modified_recipe, recommendations)
185 extensions_file = 'extensions.json'
186 IdeBase.update_json_file(
187 self.dot_code_dir(modified_recipe), extensions_file, {"recommendations": recommendations})
188
189 def vscode_c_cpp_properties(self, modified_recipe):
190 properties_dict = {
191 "name": modified_recipe.recipe_id_pretty,
192 }
193 if modified_recipe.build_tool is BuildTool.CMAKE:
194 properties_dict["configurationProvider"] = "ms-vscode.cmake-tools"
195 elif modified_recipe.build_tool is BuildTool.MESON:
196 properties_dict["configurationProvider"] = "mesonbuild.mesonbuild"
197 properties_dict["compilerPath"] = os.path.join(modified_recipe.staging_bindir_toolchain, modified_recipe.cxx.split()[0])
198 else: # no C/C++ build
199 return
200
201 properties_dicts = {
202 "configurations": [
203 properties_dict
204 ],
205 "version": 4
206 }
207 prop_file = 'c_cpp_properties.json'
208 IdeBase.update_json_file(
209 self.dot_code_dir(modified_recipe), prop_file, properties_dicts)
210
211 def vscode_launch_bin_dbg(self, gdb_cross_config):
212 modified_recipe = gdb_cross_config.modified_recipe
213
214 launch_config = {
215 "name": gdb_cross_config.id_pretty,
216 "type": "cppdbg",
217 "request": "launch",
218 "program": os.path.join(modified_recipe.d, gdb_cross_config.binary.lstrip('/')),
219 "stopAtEntry": True,
220 "cwd": "${workspaceFolder}",
221 "environment": [],
222 "externalConsole": False,
223 "MIMode": "gdb",
224 "preLaunchTask": gdb_cross_config.id_pretty,
225 "miDebuggerPath": modified_recipe.gdb_cross.gdb,
226 "miDebuggerServerAddress": "%s:%d" % (modified_recipe.gdb_cross.host, gdb_cross_config.gdbserver_port)
227 }
228
229 # Search for header files in recipe-sysroot.
230 src_file_map = {
231 "/usr/include": os.path.join(modified_recipe.recipe_sysroot, "usr", "include")
232 }
233 # First of all search for not stripped binaries in the image folder.
234 # These binaries are copied (and optionally stripped) by deploy-target
235 setup_commands = [
236 {
237 "description": "sysroot",
238 "text": "set sysroot " + modified_recipe.d
239 }
240 ]
241
242 if gdb_cross_config.image_recipe.rootfs_dbg:
243 launch_config['additionalSOLibSearchPath'] = modified_recipe.solib_search_path_str(
244 gdb_cross_config.image_recipe)
245 # First: Search for sources of this recipe in the workspace folder
246 if modified_recipe.pn in modified_recipe.target_dbgsrc_dir:
247 src_file_map[modified_recipe.target_dbgsrc_dir] = "${workspaceFolder}"
248 else:
249 logger.error(
250 "TARGET_DBGSRC_DIR must contain the recipe name PN.")
251 # Second: Search for sources of other recipes in the rootfs-dbg
252 if modified_recipe.target_dbgsrc_dir.startswith("/usr/src/debug"):
253 src_file_map["/usr/src/debug"] = os.path.join(
254 gdb_cross_config.image_recipe.rootfs_dbg, "usr", "src", "debug")
255 else:
256 logger.error(
257 "TARGET_DBGSRC_DIR must start with /usr/src/debug.")
258 else:
259 logger.warning(
260 "Cannot setup debug symbols configuration for GDB. IMAGE_GEN_DEBUGFS is not enabled.")
261
262 launch_config['sourceFileMap'] = src_file_map
263 launch_config['setupCommands'] = setup_commands
264 return launch_config
265
266 def vscode_launch(self, modified_recipe):
267 """GDB Launch configuration for binaries (elf files)"""
268
269 configurations = []
270 for gdb_cross_config in self.gdb_cross_configs:
271 if gdb_cross_config.modified_recipe is modified_recipe:
272 configurations.append(self.vscode_launch_bin_dbg(gdb_cross_config))
273 launch_dict = {
274 "version": "0.2.0",
275 "configurations": configurations
276 }
277 launch_file = 'launch.json'
278 IdeBase.update_json_file(
279 self.dot_code_dir(modified_recipe), launch_file, launch_dict)
280
281 def vscode_tasks_cpp(self, args, modified_recipe):
282 run_install_deploy = modified_recipe.gen_install_deploy_script(args)
283 install_task_name = "install && deploy-target %s" % modified_recipe.recipe_id_pretty
284 tasks_dict = {
285 "version": "2.0.0",
286 "tasks": [
287 {
288 "label": install_task_name,
289 "type": "shell",
290 "command": run_install_deploy,
291 "problemMatcher": []
292 }
293 ]
294 }
295 for gdb_cross_config in self.gdb_cross_configs:
296 if gdb_cross_config.modified_recipe is not modified_recipe:
297 continue
298 tasks_dict['tasks'].append(
299 {
300 "label": gdb_cross_config.id_pretty,
301 "type": "shell",
302 "isBackground": True,
303 "dependsOn": [
304 install_task_name
305 ],
306 "command": gdb_cross_config.gdbserver_script,
307 "problemMatcher": [
308 {
309 "pattern": [
310 {
311 "regexp": ".",
312 "file": 1,
313 "location": 2,
314 "message": 3
315 }
316 ],
317 "background": {
318 "activeOnStart": True,
319 "beginsPattern": ".",
320 "endsPattern": ".",
321 }
322 }
323 ]
324 })
325 tasks_file = 'tasks.json'
326 IdeBase.update_json_file(
327 self.dot_code_dir(modified_recipe), tasks_file, tasks_dict)
328
329 def vscode_tasks_fallback(self, args, modified_recipe):
330 oe_init_dir = modified_recipe.oe_init_dir
331 oe_init = ". %s %s > /dev/null && " % (modified_recipe.oe_init_build_env, modified_recipe.topdir)
332 dt_build = "devtool build "
333 dt_build_label = dt_build + modified_recipe.recipe_id_pretty
334 dt_build_cmd = dt_build + modified_recipe.bpn
335 clean_opt = " --clean"
336 dt_build_clean_label = dt_build + modified_recipe.recipe_id_pretty + clean_opt
337 dt_build_clean_cmd = dt_build + modified_recipe.bpn + clean_opt
338 dt_deploy = "devtool deploy-target "
339 dt_deploy_label = dt_deploy + modified_recipe.recipe_id_pretty
340 dt_deploy_cmd = dt_deploy + modified_recipe.bpn
341 dt_build_deploy_label = "devtool build & deploy-target %s" % modified_recipe.recipe_id_pretty
342 deploy_opts = ' '.join(get_devtool_deploy_opts(args))
343 tasks_dict = {
344 "version": "2.0.0",
345 "tasks": [
346 {
347 "label": dt_build_label,
348 "type": "shell",
349 "command": "bash",
350 "linux": {
351 "options": {
352 "cwd": oe_init_dir
353 }
354 },
355 "args": [
356 "--login",
357 "-c",
358 "%s%s" % (oe_init, dt_build_cmd)
359 ],
360 "problemMatcher": []
361 },
362 {
363 "label": dt_deploy_label,
364 "type": "shell",
365 "command": "bash",
366 "linux": {
367 "options": {
368 "cwd": oe_init_dir
369 }
370 },
371 "args": [
372 "--login",
373 "-c",
374 "%s%s %s" % (
375 oe_init, dt_deploy_cmd, deploy_opts)
376 ],
377 "problemMatcher": []
378 },
379 {
380 "label": dt_build_deploy_label,
381 "dependsOrder": "sequence",
382 "dependsOn": [
383 dt_build_label,
384 dt_deploy_label
385 ],
386 "problemMatcher": [],
387 "group": {
388 "kind": "build",
389 "isDefault": True
390 }
391 },
392 {
393 "label": dt_build_clean_label,
394 "type": "shell",
395 "command": "bash",
396 "linux": {
397 "options": {
398 "cwd": oe_init_dir
399 }
400 },
401 "args": [
402 "--login",
403 "-c",
404 "%s%s" % (oe_init, dt_build_clean_cmd)
405 ],
406 "problemMatcher": []
407 }
408 ]
409 }
410 if modified_recipe.gdb_cross:
411 for gdb_cross_config in self.gdb_cross_configs:
412 if gdb_cross_config.modified_recipe is not modified_recipe:
413 continue
414 tasks_dict['tasks'].append(
415 {
416 "label": gdb_cross_config.id_pretty,
417 "type": "shell",
418 "isBackground": True,
419 "dependsOn": [
420 dt_build_deploy_label
421 ],
422 "command": gdb_cross_config.gdbserver_script,
423 "problemMatcher": [
424 {
425 "pattern": [
426 {
427 "regexp": ".",
428 "file": 1,
429 "location": 2,
430 "message": 3
431 }
432 ],
433 "background": {
434 "activeOnStart": True,
435 "beginsPattern": ".",
436 "endsPattern": ".",
437 }
438 }
439 ]
440 })
441 tasks_file = 'tasks.json'
442 IdeBase.update_json_file(
443 self.dot_code_dir(modified_recipe), tasks_file, tasks_dict)
444
445 def vscode_tasks(self, args, modified_recipe):
446 if modified_recipe.build_tool.is_c_ccp:
447 self.vscode_tasks_cpp(args, modified_recipe)
448 else:
449 self.vscode_tasks_fallback(args, modified_recipe)
450
451 def setup_modified_recipe(self, args, image_recipe, modified_recipe):
452 self.vscode_settings(modified_recipe, image_recipe)
453 self.vscode_extensions(modified_recipe)
454 self.vscode_c_cpp_properties(modified_recipe)
455 if args.target:
456 self.initialize_gdb_cross_configs(
457 image_recipe, modified_recipe, gdb_cross_config_class=GdbCrossConfigVSCode)
458 self.vscode_launch(modified_recipe)
459 self.vscode_tasks(args, modified_recipe)
460
461
462def register_ide_plugin(ide_plugins):
463 ide_plugins['code'] = IdeVSCode
diff --git a/scripts/lib/devtool/ide_plugins/ide_none.py b/scripts/lib/devtool/ide_plugins/ide_none.py
new file mode 100644
index 0000000000..f106c5a026
--- /dev/null
+++ b/scripts/lib/devtool/ide_plugins/ide_none.py
@@ -0,0 +1,53 @@
1#
2# Copyright (C) 2023-2024 Siemens AG
3#
4# SPDX-License-Identifier: GPL-2.0-only
5#
6"""Devtool ide-sdk generic IDE plugin"""
7
8import os
9import logging
10from devtool.ide_plugins import IdeBase, GdbCrossConfig
11
12logger = logging.getLogger('devtool')
13
14
15class IdeNone(IdeBase):
16 """Generate some generic helpers for other IDEs
17
18 Modified recipe mode:
19 Generate some helper scripts for remote debugging with GDB
20
21 Shared sysroot mode:
22 A wrapper for bitbake meta-ide-support and bitbake build-sysroots
23 """
24
25 def __init__(self):
26 super().__init__()
27
28 def setup_shared_sysroots(self, shared_env):
29 real_multimach_target_sys = shared_env.ide_support.real_multimach_target_sys
30 deploy_dir_image = shared_env.ide_support.deploy_dir_image
31 env_script = os.path.join(
32 deploy_dir_image, 'environment-setup-' + real_multimach_target_sys)
33 logger.info(
34 "To use this SDK please source this: %s" % env_script)
35
36 def setup_modified_recipe(self, args, image_recipe, modified_recipe):
37 """generate some helper scripts and config files
38
39 - Execute the do_install task
40 - Execute devtool deploy-target
41 - Generate a gdbinit file per executable
42 - Generate the oe-scripts sym-link
43 """
44 script_path = modified_recipe.gen_install_deploy_script(args)
45 logger.info("Created: %s" % script_path)
46
47 self.initialize_gdb_cross_configs(image_recipe, modified_recipe)
48
49 IdeBase.gen_oe_scrtips_sym_link(modified_recipe)
50
51
52def register_ide_plugin(ide_plugins):
53 ide_plugins['none'] = IdeNone
diff --git a/scripts/lib/devtool/ide_sdk.py b/scripts/lib/devtool/ide_sdk.py
new file mode 100755
index 0000000000..7807b322b3
--- /dev/null
+++ b/scripts/lib/devtool/ide_sdk.py
@@ -0,0 +1,1070 @@
1# Development tool - ide-sdk command plugin
2#
3# Copyright (C) 2023-2024 Siemens AG
4#
5# SPDX-License-Identifier: GPL-2.0-only
6#
7"""Devtool ide-sdk plugin"""
8
9import json
10import logging
11import os
12import re
13import shutil
14import stat
15import subprocess
16import sys
17from argparse import RawTextHelpFormatter
18from enum import Enum
19
20import scriptutils
21import bb
22from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, DevtoolError, parse_recipe
23from devtool.standard import get_real_srctree
24from devtool.ide_plugins import BuildTool
25
26
27logger = logging.getLogger('devtool')
28
29# dict of classes derived from IdeBase
30ide_plugins = {}
31
32
33class DevtoolIdeMode(Enum):
34 """Different modes are supported by the ide-sdk plugin.
35
36 The enum might be extended by more advanced modes in the future. Some ideas:
37 - auto: modified if all recipes are modified, shared if none of the recipes is modified.
38 - mixed: modified mode for modified recipes, shared mode for all other recipes.
39 """
40
41 modified = 'modified'
42 shared = 'shared'
43
44
45class TargetDevice:
46 """SSH remote login parameters"""
47
48 def __init__(self, args):
49 self.extraoptions = ''
50 if args.no_host_check:
51 self.extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
52 self.ssh_sshexec = 'ssh'
53 if args.ssh_exec:
54 self.ssh_sshexec = args.ssh_exec
55 self.ssh_port = ''
56 if args.port:
57 self.ssh_port = "-p %s" % args.port
58 if args.key:
59 self.extraoptions += ' -i %s' % args.key
60
61 self.target = args.target
62 target_sp = args.target.split('@')
63 if len(target_sp) == 1:
64 self.login = ""
65 self.host = target_sp[0]
66 elif len(target_sp) == 2:
67 self.login = target_sp[0]
68 self.host = target_sp[1]
69 else:
70 logger.error("Invalid target argument: %s" % args.target)
71
72
73class RecipeNative:
74 """Base class for calling bitbake to provide a -native recipe"""
75
76 def __init__(self, name, target_arch=None):
77 self.name = name
78 self.target_arch = target_arch
79 self.bootstrap_tasks = [self.name + ':do_addto_recipe_sysroot']
80 self.staging_bindir_native = None
81 self.target_sys = None
82 self.__native_bin = None
83
84 def _initialize(self, config, workspace, tinfoil):
85 """Get the parsed recipe"""
86 recipe_d = parse_recipe(
87 config, tinfoil, self.name, appends=True, filter_workspace=False)
88 if not recipe_d:
89 raise DevtoolError("Parsing %s recipe failed" % self.name)
90 self.staging_bindir_native = os.path.realpath(
91 recipe_d.getVar('STAGING_BINDIR_NATIVE'))
92 self.target_sys = recipe_d.getVar('TARGET_SYS')
93 return recipe_d
94
95 def initialize(self, config, workspace, tinfoil):
96 """Basic initialization that can be overridden by a derived class"""
97 self._initialize(config, workspace, tinfoil)
98
99 @property
100 def native_bin(self):
101 if not self.__native_bin:
102 raise DevtoolError("native binary name is not defined.")
103 return self.__native_bin
104
105
106class RecipeGdbCross(RecipeNative):
107 """Handle handle gdb-cross on the host and the gdbserver on the target device"""
108
109 def __init__(self, args, target_arch, target_device):
110 super().__init__('gdb-cross-' + target_arch, target_arch)
111 self.target_device = target_device
112 self.gdb = None
113 self.gdbserver_port_next = int(args.gdbserver_port_start)
114 self.config_db = {}
115
116 def __find_gdbserver(self, config, tinfoil):
117 """Absolute path of the gdbserver"""
118 recipe_d_gdb = parse_recipe(
119 config, tinfoil, 'gdb', appends=True, filter_workspace=False)
120 if not recipe_d_gdb:
121 raise DevtoolError("Parsing gdb recipe failed")
122 return os.path.join(recipe_d_gdb.getVar('bindir'), 'gdbserver')
123
124 def initialize(self, config, workspace, tinfoil):
125 super()._initialize(config, workspace, tinfoil)
126 gdb_bin = self.target_sys + '-gdb'
127 gdb_path = os.path.join(
128 self.staging_bindir_native, self.target_sys, gdb_bin)
129 self.gdb = gdb_path
130 self.gdbserver_path = self.__find_gdbserver(config, tinfoil)
131
132 @property
133 def host(self):
134 return self.target_device.host
135
136
137class RecipeImage:
138 """Handle some image recipe related properties
139
140 Most workflows require firmware that runs on the target device.
141 This firmware must be consistent with the setup of the host system.
142 In particular, the debug symbols must be compatible. For this, the
143 rootfs must be created as part of the SDK.
144 """
145
146 def __init__(self, name):
147 self.combine_dbg_image = False
148 self.gdbserver_missing = False
149 self.name = name
150 self.rootfs = None
151 self.__rootfs_dbg = None
152 self.bootstrap_tasks = [self.name + ':do_build']
153
154 def initialize(self, config, tinfoil):
155 image_d = parse_recipe(
156 config, tinfoil, self.name, appends=True, filter_workspace=False)
157 if not image_d:
158 raise DevtoolError(
159 "Parsing image recipe %s failed" % self.name)
160
161 self.combine_dbg_image = bb.data.inherits_class(
162 'image-combined-dbg', image_d)
163
164 workdir = image_d.getVar('WORKDIR')
165 self.rootfs = os.path.join(workdir, 'rootfs')
166 if image_d.getVar('IMAGE_GEN_DEBUGFS') == "1":
167 self.__rootfs_dbg = os.path.join(workdir, 'rootfs-dbg')
168
169 self.gdbserver_missing = 'gdbserver' not in image_d.getVar(
170 'IMAGE_INSTALL')
171
172 @property
173 def debug_support(self):
174 return bool(self.rootfs_dbg)
175
176 @property
177 def rootfs_dbg(self):
178 if self.__rootfs_dbg and os.path.isdir(self.__rootfs_dbg):
179 return self.__rootfs_dbg
180 return None
181
182
183class RecipeMetaIdeSupport:
184 """For the shared sysroots mode meta-ide-support is needed
185
186 For use cases where just a cross tool-chain is required but
187 no recipe is used, devtool ide-sdk abstracts calling bitbake meta-ide-support
188 and bitbake build-sysroots. This also allows to expose the cross-toolchains
189 to IDEs. For example VSCode support different tool-chains with e.g. cmake-kits.
190 """
191
192 def __init__(self):
193 self.bootstrap_tasks = ['meta-ide-support:do_build']
194 self.topdir = None
195 self.datadir = None
196 self.deploy_dir_image = None
197 self.build_sys = None
198 # From toolchain-scripts
199 self.real_multimach_target_sys = None
200
201 def initialize(self, config, tinfoil):
202 meta_ide_support_d = parse_recipe(
203 config, tinfoil, 'meta-ide-support', appends=True, filter_workspace=False)
204 if not meta_ide_support_d:
205 raise DevtoolError("Parsing meta-ide-support recipe failed")
206
207 self.topdir = meta_ide_support_d.getVar('TOPDIR')
208 self.datadir = meta_ide_support_d.getVar('datadir')
209 self.deploy_dir_image = meta_ide_support_d.getVar(
210 'DEPLOY_DIR_IMAGE')
211 self.build_sys = meta_ide_support_d.getVar('BUILD_SYS')
212 self.real_multimach_target_sys = meta_ide_support_d.getVar(
213 'REAL_MULTIMACH_TARGET_SYS')
214
215
216class RecipeBuildSysroots:
217 """For the shared sysroots mode build-sysroots is needed"""
218
219 def __init__(self):
220 self.standalone_sysroot = None
221 self.standalone_sysroot_native = None
222 self.bootstrap_tasks = [
223 'build-sysroots:do_build_target_sysroot',
224 'build-sysroots:do_build_native_sysroot'
225 ]
226
227 def initialize(self, config, tinfoil):
228 build_sysroots_d = parse_recipe(
229 config, tinfoil, 'build-sysroots', appends=True, filter_workspace=False)
230 if not build_sysroots_d:
231 raise DevtoolError("Parsing build-sysroots recipe failed")
232 self.standalone_sysroot = build_sysroots_d.getVar(
233 'STANDALONE_SYSROOT')
234 self.standalone_sysroot_native = build_sysroots_d.getVar(
235 'STANDALONE_SYSROOT_NATIVE')
236
237
238class SharedSysrootsEnv:
239 """Handle the shared sysroots based workflow
240
241 Support the workflow with just a tool-chain without a recipe.
242 It's basically like:
243 bitbake some-dependencies
244 bitbake meta-ide-support
245 bitbake build-sysroots
246 Use the environment-* file found in the deploy folder
247 """
248
249 def __init__(self):
250 self.ide_support = None
251 self.build_sysroots = None
252
253 def initialize(self, ide_support, build_sysroots):
254 self.ide_support = ide_support
255 self.build_sysroots = build_sysroots
256
257 def setup_ide(self, ide):
258 ide.setup(self)
259
260
261class RecipeNotModified:
262 """Handling of recipes added to the Direct DSK shared sysroots."""
263
264 def __init__(self, name):
265 self.name = name
266 self.bootstrap_tasks = [name + ':do_populate_sysroot']
267
268
269class RecipeModified:
270 """Handling of recipes in the workspace created by devtool modify"""
271 OE_INIT_BUILD_ENV = 'oe-init-build-env'
272
273 VALID_BASH_ENV_NAME_CHARS = re.compile(r"^[a-zA-Z0-9_]*$")
274
275 def __init__(self, name):
276 self.name = name
277 self.bootstrap_tasks = [name + ':do_install']
278 self.gdb_cross = None
279 # workspace
280 self.real_srctree = None
281 self.srctree = None
282 self.ide_sdk_dir = None
283 self.ide_sdk_scripts_dir = None
284 self.bbappend = None
285 # recipe variables from d.getVar
286 self.b = None
287 self.base_libdir = None
288 self.bblayers = None
289 self.bpn = None
290 self.d = None
291 self.fakerootcmd = None
292 self.fakerootenv = None
293 self.libdir = None
294 self.max_process = None
295 self.package_arch = None
296 self.package_debug_split_style = None
297 self.path = None
298 self.pn = None
299 self.recipe_sysroot = None
300 self.recipe_sysroot_native = None
301 self.staging_incdir = None
302 self.strip_cmd = None
303 self.target_arch = None
304 self.target_dbgsrc_dir = None
305 self.topdir = None
306 self.workdir = None
307 self.recipe_id = None
308 # replicate bitbake build environment
309 self.exported_vars = None
310 self.cmd_compile = None
311 self.__oe_init_dir = None
312 # main build tool used by this recipe
313 self.build_tool = BuildTool.UNDEFINED
314 # build_tool = cmake
315 self.oecmake_generator = None
316 self.cmake_cache_vars = None
317 # build_tool = meson
318 self.meson_buildtype = None
319 self.meson_wrapper = None
320 self.mesonopts = None
321 self.extra_oemeson = None
322 self.meson_cross_file = None
323
324 def initialize(self, config, workspace, tinfoil):
325 recipe_d = parse_recipe(
326 config, tinfoil, self.name, appends=True, filter_workspace=False)
327 if not recipe_d:
328 raise DevtoolError("Parsing %s recipe failed" % self.name)
329
330 # Verify this recipe is built as externalsrc setup by devtool modify
331 workspacepn = check_workspace_recipe(
332 workspace, self.name, bbclassextend=True)
333 self.srctree = workspace[workspacepn]['srctree']
334 # Need to grab this here in case the source is within a subdirectory
335 self.real_srctree = get_real_srctree(
336 self.srctree, recipe_d.getVar('S'), recipe_d.getVar('WORKDIR'))
337 self.bbappend = workspace[workspacepn]['bbappend']
338
339 self.ide_sdk_dir = os.path.join(
340 config.workspace_path, 'ide-sdk', self.name)
341 if os.path.exists(self.ide_sdk_dir):
342 shutil.rmtree(self.ide_sdk_dir)
343 self.ide_sdk_scripts_dir = os.path.join(self.ide_sdk_dir, 'scripts')
344
345 self.b = recipe_d.getVar('B')
346 self.base_libdir = recipe_d.getVar('base_libdir')
347 self.bblayers = recipe_d.getVar('BBLAYERS').split()
348 self.bpn = recipe_d.getVar('BPN')
349 self.cxx = recipe_d.getVar('CXX')
350 self.d = recipe_d.getVar('D')
351 self.fakerootcmd = recipe_d.getVar('FAKEROOTCMD')
352 self.fakerootenv = recipe_d.getVar('FAKEROOTENV')
353 self.libdir = recipe_d.getVar('libdir')
354 self.max_process = int(recipe_d.getVar(
355 "BB_NUMBER_THREADS") or os.cpu_count() or 1)
356 self.package_arch = recipe_d.getVar('PACKAGE_ARCH')
357 self.package_debug_split_style = recipe_d.getVar(
358 'PACKAGE_DEBUG_SPLIT_STYLE')
359 self.path = recipe_d.getVar('PATH')
360 self.pn = recipe_d.getVar('PN')
361 self.recipe_sysroot = os.path.realpath(
362 recipe_d.getVar('RECIPE_SYSROOT'))
363 self.recipe_sysroot_native = os.path.realpath(
364 recipe_d.getVar('RECIPE_SYSROOT_NATIVE'))
365 self.staging_bindir_toolchain = os.path.realpath(
366 recipe_d.getVar('STAGING_BINDIR_TOOLCHAIN'))
367 self.staging_incdir = os.path.realpath(
368 recipe_d.getVar('STAGING_INCDIR'))
369 self.strip_cmd = recipe_d.getVar('STRIP')
370 self.target_arch = recipe_d.getVar('TARGET_ARCH')
371 self.target_dbgsrc_dir = recipe_d.getVar('TARGET_DBGSRC_DIR')
372 self.topdir = recipe_d.getVar('TOPDIR')
373 self.workdir = os.path.realpath(recipe_d.getVar('WORKDIR'))
374
375 self.__init_exported_variables(recipe_d)
376
377 if bb.data.inherits_class('cmake', recipe_d):
378 self.oecmake_generator = recipe_d.getVar('OECMAKE_GENERATOR')
379 self.__init_cmake_preset_cache(recipe_d)
380 self.build_tool = BuildTool.CMAKE
381 elif bb.data.inherits_class('meson', recipe_d):
382 self.meson_buildtype = recipe_d.getVar('MESON_BUILDTYPE')
383 self.mesonopts = recipe_d.getVar('MESONOPTS')
384 self.extra_oemeson = recipe_d.getVar('EXTRA_OEMESON')
385 self.meson_cross_file = recipe_d.getVar('MESON_CROSS_FILE')
386 self.build_tool = BuildTool.MESON
387
388 # Recipe ID is the identifier for IDE config sections
389 self.recipe_id = self.bpn + "-" + self.package_arch
390 self.recipe_id_pretty = self.bpn + ": " + self.package_arch
391
392 def append_to_bbappend(self, append_text):
393 with open(self.bbappend, 'a') as bbap:
394 bbap.write(append_text)
395
396 def remove_from_bbappend(self, append_text):
397 with open(self.bbappend, 'r') as bbap:
398 text = bbap.read()
399 new_text = text.replace(append_text, '')
400 with open(self.bbappend, 'w') as bbap:
401 bbap.write(new_text)
402
403 @staticmethod
404 def is_valid_shell_variable(var):
405 """Skip strange shell variables like systemd
406
407 prevent from strange bugs because of strange variables which
408 are not used in this context but break various tools.
409 """
410 if RecipeModified.VALID_BASH_ENV_NAME_CHARS.match(var):
411 bb.debug(1, "ignoring variable: %s" % var)
412 return True
413 return False
414
415 def debug_build_config(self, args):
416 """Explicitely set for example CMAKE_BUILD_TYPE to Debug if not defined otherwise"""
417 if self.build_tool is BuildTool.CMAKE:
418 append_text = os.linesep + \
419 'OECMAKE_ARGS:append = " -DCMAKE_BUILD_TYPE:STRING=Debug"' + os.linesep
420 if args.debug_build_config and not 'CMAKE_BUILD_TYPE' in self.cmake_cache_vars:
421 self.cmake_cache_vars['CMAKE_BUILD_TYPE'] = {
422 "type": "STRING",
423 "value": "Debug",
424 }
425 self.append_to_bbappend(append_text)
426 elif 'CMAKE_BUILD_TYPE' in self.cmake_cache_vars:
427 del self.cmake_cache_vars['CMAKE_BUILD_TYPE']
428 self.remove_from_bbappend(append_text)
429 elif self.build_tool is BuildTool.MESON:
430 append_text = os.linesep + 'MESON_BUILDTYPE = "debug"' + os.linesep
431 if args.debug_build_config and self.meson_buildtype != "debug":
432 self.mesonopts.replace(
433 '--buildtype ' + self.meson_buildtype, '--buildtype debug')
434 self.append_to_bbappend(append_text)
435 elif self.meson_buildtype == "debug":
436 self.mesonopts.replace(
437 '--buildtype debug', '--buildtype plain')
438 self.remove_from_bbappend(append_text)
439 elif args.debug_build_config:
440 logger.warn(
441 "--debug-build-config is not implemented for this build tool yet.")
442
443 def solib_search_path(self, image):
444 """Search for debug symbols in the rootfs and rootfs-dbg
445
446 The debug symbols of shared libraries which are provided by other packages
447 are grabbed from the -dbg packages in the rootfs-dbg.
448
449 But most cross debugging tools like gdb, perf, and systemtap need to find
450 executable/library first and through it debuglink note find corresponding
451 symbols file. Therefore the library paths from the rootfs are added as well.
452
453 Note: For the devtool modified recipe compiled from the IDE, the debug
454 symbols are taken from the unstripped binaries in the image folder.
455 Also, devtool deploy-target takes the files from the image folder.
456 debug symbols in the image folder refer to the corresponding source files
457 with absolute paths of the build machine. Debug symbols found in the
458 rootfs-dbg are relocated and contain paths which refer to the source files
459 installed on the target device e.g. /usr/src/...
460 """
461 base_libdir = self.base_libdir.lstrip('/')
462 libdir = self.libdir.lstrip('/')
463 so_paths = [
464 # debug symbols for package_debug_split_style: debug-with-srcpkg or .debug
465 os.path.join(image.rootfs_dbg, base_libdir, ".debug"),
466 os.path.join(image.rootfs_dbg, libdir, ".debug"),
467 # debug symbols for package_debug_split_style: debug-file-directory
468 os.path.join(image.rootfs_dbg, "usr", "lib", "debug"),
469
470 # The binaries are required as well, the debug packages are not enough
471 # With image-combined-dbg.bbclass the binaries are copied into rootfs-dbg
472 os.path.join(image.rootfs_dbg, base_libdir),
473 os.path.join(image.rootfs_dbg, libdir),
474 # Without image-combined-dbg.bbclass the binaries are only in rootfs.
475 # Note: Stepping into source files located in rootfs-dbg does not
476 # work without image-combined-dbg.bbclass yet.
477 os.path.join(image.rootfs, base_libdir),
478 os.path.join(image.rootfs, libdir)
479 ]
480 return so_paths
481
482 def solib_search_path_str(self, image):
483 """Return a : separated list of paths usable by GDB's set solib-search-path"""
484 return ':'.join(self.solib_search_path(image))
485
486 def __init_exported_variables(self, d):
487 """Find all variables with export flag set.
488
489 This allows to generate IDE configurations which compile with the same
490 environment as bitbake does. That's at least a reasonable default behavior.
491 """
492 exported_vars = {}
493
494 vars = (key for key in d.keys() if not key.startswith(
495 "__") and not d.getVarFlag(key, "func", False))
496 for var in vars:
497 func = d.getVarFlag(var, "func", False)
498 if d.getVarFlag(var, 'python', False) and func:
499 continue
500 export = d.getVarFlag(var, "export", False)
501 unexport = d.getVarFlag(var, "unexport", False)
502 if not export and not unexport and not func:
503 continue
504 if unexport:
505 continue
506
507 val = d.getVar(var)
508 if val is None:
509 continue
510 if set(var) & set("-.{}+"):
511 logger.warn(
512 "Warning: Found invalid character in variable name %s", str(var))
513 continue
514 varExpanded = d.expand(var)
515 val = str(val)
516
517 if not RecipeModified.is_valid_shell_variable(varExpanded):
518 continue
519
520 if func:
521 code_line = "line: {0}, file: {1}\n".format(
522 d.getVarFlag(var, "lineno", False),
523 d.getVarFlag(var, "filename", False))
524 val = val.rstrip('\n')
525 logger.warn("Warning: exported shell function %s() is not exported (%s)" %
526 (varExpanded, code_line))
527 continue
528
529 if export:
530 exported_vars[varExpanded] = val.strip()
531 continue
532
533 self.exported_vars = exported_vars
534
535 def __init_cmake_preset_cache(self, d):
536 """Get the arguments passed to cmake
537
538 Replicate the cmake configure arguments with all details to
539 share on build folder between bitbake and SDK.
540 """
541 site_file = os.path.join(self.workdir, 'site-file.cmake')
542 if os.path.exists(site_file):
543 print("Warning: site-file.cmake is not supported")
544
545 cache_vars = {}
546 oecmake_args = d.getVar('OECMAKE_ARGS').split()
547 extra_oecmake = d.getVar('EXTRA_OECMAKE').split()
548 for param in oecmake_args + extra_oecmake:
549 d_pref = "-D"
550 if param.startswith(d_pref):
551 param = param[len(d_pref):]
552 else:
553 print("Error: expected a -D")
554 param_s = param.split('=', 1)
555 param_nt = param_s[0].split(':', 1)
556
557 def handle_undefined_variable(var):
558 if var.startswith('${') and var.endswith('}'):
559 return ''
560 else:
561 return var
562 # Example: FOO=ON
563 if len(param_nt) == 1:
564 cache_vars[param_s[0]] = handle_undefined_variable(param_s[1])
565 # Example: FOO:PATH=/tmp
566 elif len(param_nt) == 2:
567 cache_vars[param_nt[0]] = {
568 "type": param_nt[1],
569 "value": handle_undefined_variable(param_s[1]),
570 }
571 else:
572 print("Error: cannot parse %s" % param)
573 self.cmake_cache_vars = cache_vars
574
575 def cmake_preset(self):
576 """Create a preset for cmake that mimics how bitbake calls cmake"""
577 toolchain_file = os.path.join(self.workdir, 'toolchain.cmake')
578 cmake_executable = os.path.join(
579 self.recipe_sysroot_native, 'usr', 'bin', 'cmake')
580 self.cmd_compile = cmake_executable + " --build --preset " + self.recipe_id
581
582 preset_dict_configure = {
583 "name": self.recipe_id,
584 "displayName": self.recipe_id_pretty,
585 "description": "Bitbake build environment for the recipe %s compiled for %s" % (self.bpn, self.package_arch),
586 "binaryDir": self.b,
587 "generator": self.oecmake_generator,
588 "toolchainFile": toolchain_file,
589 "cacheVariables": self.cmake_cache_vars,
590 "environment": self.exported_vars,
591 "cmakeExecutable": cmake_executable
592 }
593
594 preset_dict_build = {
595 "name": self.recipe_id,
596 "displayName": self.recipe_id_pretty,
597 "description": "Bitbake build environment for the recipe %s compiled for %s" % (self.bpn, self.package_arch),
598 "configurePreset": self.recipe_id,
599 "inheritConfigureEnvironment": True
600 }
601
602 preset_dict_test = {
603 "name": self.recipe_id,
604 "displayName": self.recipe_id_pretty,
605 "description": "Bitbake build environment for the recipe %s compiled for %s" % (self.bpn, self.package_arch),
606 "configurePreset": self.recipe_id,
607 "inheritConfigureEnvironment": True
608 }
609
610 preset_dict = {
611 "version": 3, # cmake 3.21, backward compatible with kirkstone
612 "configurePresets": [preset_dict_configure],
613 "buildPresets": [preset_dict_build],
614 "testPresets": [preset_dict_test]
615 }
616
617 # Finally write the json file
618 json_file = 'CMakeUserPresets.json'
619 json_path = os.path.join(self.real_srctree, json_file)
620 logger.info("Updating CMake preset: %s (%s)" % (json_file, json_path))
621 if not os.path.exists(self.real_srctree):
622 os.makedirs(self.real_srctree)
623 try:
624 with open(json_path) as f:
625 orig_dict = json.load(f)
626 except json.decoder.JSONDecodeError:
627 logger.info(
628 "Decoding %s failed. Probably because of comments in the json file" % json_path)
629 orig_dict = {}
630 except FileNotFoundError:
631 orig_dict = {}
632
633 # Add or update the presets for the recipe and keep other presets
634 for k, v in preset_dict.items():
635 if isinstance(v, list):
636 update_preset = v[0]
637 preset_added = False
638 if k in orig_dict:
639 for index, orig_preset in enumerate(orig_dict[k]):
640 if 'name' in orig_preset:
641 if orig_preset['name'] == update_preset['name']:
642 logger.debug("Updating preset: %s" %
643 orig_preset['name'])
644 orig_dict[k][index] = update_preset
645 preset_added = True
646 break
647 else:
648 logger.debug("keeping preset: %s" %
649 orig_preset['name'])
650 else:
651 logger.warn("preset without a name found")
652 if not preset_added:
653 if not k in orig_dict:
654 orig_dict[k] = []
655 orig_dict[k].append(update_preset)
656 logger.debug("Added preset: %s" %
657 update_preset['name'])
658 else:
659 orig_dict[k] = v
660
661 with open(json_path, 'w') as f:
662 json.dump(orig_dict, f, indent=4)
663
664 def gen_meson_wrapper(self):
665 """Generate a wrapper script to call meson with the cross environment"""
666 bb.utils.mkdirhier(self.ide_sdk_scripts_dir)
667 meson_wrapper = os.path.join(self.ide_sdk_scripts_dir, 'meson')
668 meson_real = os.path.join(
669 self.recipe_sysroot_native, 'usr', 'bin', 'meson.real')
670 with open(meson_wrapper, 'w') as mwrap:
671 mwrap.write("#!/bin/sh" + os.linesep)
672 for var, val in self.exported_vars.items():
673 mwrap.write('export %s="%s"' % (var, val) + os.linesep)
674 mwrap.write("unset CC CXX CPP LD AR NM STRIP" + os.linesep)
675 private_temp = os.path.join(self.b, "meson-private", "tmp")
676 mwrap.write('mkdir -p "%s"' % private_temp + os.linesep)
677 mwrap.write('export TMPDIR="%s"' % private_temp + os.linesep)
678 mwrap.write('exec "%s" "$@"' % meson_real + os.linesep)
679 st = os.stat(meson_wrapper)
680 os.chmod(meson_wrapper, st.st_mode | stat.S_IEXEC)
681 self.meson_wrapper = meson_wrapper
682 self.cmd_compile = meson_wrapper + " compile -C " + self.b
683
684 def which(self, executable):
685 bin_path = shutil.which(executable, path=self.path)
686 if not bin_path:
687 raise DevtoolError(
688 'Cannot find %s. Probably the recipe %s is not built yet.' % (executable, self.bpn))
689 return bin_path
690
691 @staticmethod
692 def is_elf_file(file_path):
693 with open(file_path, "rb") as f:
694 data = f.read(4)
695 if data == b'\x7fELF':
696 return True
697 return False
698
699 def find_installed_binaries(self):
700 """find all executable elf files in the image directory"""
701 binaries = []
702 d_len = len(self.d)
703 re_so = re.compile(r'.*\.so[.0-9]*$')
704 for root, _, files in os.walk(self.d, followlinks=False):
705 for file in files:
706 if os.path.islink(file):
707 continue
708 if re_so.match(file):
709 continue
710 abs_name = os.path.join(root, file)
711 if os.access(abs_name, os.X_OK) and RecipeModified.is_elf_file(abs_name):
712 binaries.append(abs_name[d_len:])
713 return sorted(binaries)
714
715 def gen_delete_package_dirs(self):
716 """delete folders of package tasks
717
718 This is a workaround for and issue with recipes having their sources
719 downloaded as file://
720 This likely breaks pseudo like:
721 path mismatch [3 links]: ino 79147802 db
722 .../build/tmp/.../cmake-example/1.0/package/usr/src/debug/
723 cmake-example/1.0-r0/oe-local-files/cpp-example-lib.cpp
724 .../build/workspace/sources/cmake-example/oe-local-files/cpp-example-lib.cpp
725 Since the files are anyway outdated lets deleted them (also from pseudo's db) to workaround this issue.
726 """
727 cmd_lines = ['#!/bin/sh']
728
729 # Set up the appropriate environment
730 newenv = dict(os.environ)
731 for varvalue in self.fakerootenv.split():
732 if '=' in varvalue:
733 splitval = varvalue.split('=', 1)
734 newenv[splitval[0]] = splitval[1]
735
736 # Replicate the environment variables from bitbake
737 for var, val in newenv.items():
738 if not RecipeModified.is_valid_shell_variable(var):
739 continue
740 cmd_lines.append('%s="%s"' % (var, val))
741 cmd_lines.append('export %s' % var)
742
743 # Delete the folders
744 pkg_dirs = ' '.join([os.path.join(self.workdir, d) for d in [
745 "package", "packages-split", "pkgdata", "sstate-install-package", "debugsources.list", "*.spec"]])
746 cmd = "%s rm -rf %s" % (self.fakerootcmd, pkg_dirs)
747 cmd_lines.append('%s || { "%s failed"; exit 1; }' % (cmd, cmd))
748
749 return self.write_script(cmd_lines, 'delete_package_dirs')
750
751 def gen_deploy_target_script(self, args):
752 """Generate a script which does what devtool deploy-target does
753
754 This script is much quicker than devtool target-deploy. Because it
755 does not need to start a bitbake server. All information from tinfoil
756 is hard-coded in the generated script.
757 """
758 cmd_lines = ['#!%s' % str(sys.executable)]
759 cmd_lines.append('import sys')
760 cmd_lines.append('devtool_sys_path = %s' % str(sys.path))
761 cmd_lines.append('devtool_sys_path.reverse()')
762 cmd_lines.append('for p in devtool_sys_path:')
763 cmd_lines.append(' if p not in sys.path:')
764 cmd_lines.append(' sys.path.insert(0, p)')
765 cmd_lines.append('from devtool.deploy import deploy_no_d')
766 args_filter = ['debug', 'dry_run', 'key', 'no_check_space', 'no_host_check',
767 'no_preserve', 'port', 'show_status', 'ssh_exec', 'strip', 'target']
768 filtered_args_dict = {key: value for key, value in vars(
769 args).items() if key in args_filter}
770 cmd_lines.append('filtered_args_dict = %s' % str(filtered_args_dict))
771 cmd_lines.append('class Dict2Class(object):')
772 cmd_lines.append(' def __init__(self, my_dict):')
773 cmd_lines.append(' for key in my_dict:')
774 cmd_lines.append(' setattr(self, key, my_dict[key])')
775 cmd_lines.append('filtered_args = Dict2Class(filtered_args_dict)')
776 cmd_lines.append(
777 'setattr(filtered_args, "recipename", "%s")' % self.bpn)
778 cmd_lines.append('deploy_no_d("%s", "%s", "%s", "%s", "%s", "%s", %d, "%s", "%s", filtered_args)' %
779 (self.d, self.workdir, self.path, self.strip_cmd,
780 self.libdir, self.base_libdir, self.max_process,
781 self.fakerootcmd, self.fakerootenv))
782 return self.write_script(cmd_lines, 'deploy_target')
783
784 def gen_install_deploy_script(self, args):
785 """Generate a script which does install and deploy"""
786 cmd_lines = ['#!/bin/bash']
787
788 cmd_lines.append(self.gen_delete_package_dirs())
789
790 # . oe-init-build-env $BUILDDIR
791 # Note: Sourcing scripts with arguments requires bash
792 cmd_lines.append('cd "%s" || { echo "cd %s failed"; exit 1; }' % (
793 self.oe_init_dir, self.oe_init_dir))
794 cmd_lines.append('. "%s" "%s" || { echo ". %s %s failed"; exit 1; }' % (
795 self.oe_init_build_env, self.topdir, self.oe_init_build_env, self.topdir))
796
797 # bitbake -c install
798 cmd_lines.append(
799 'bitbake %s -c install --force || { echo "bitbake %s -c install --force failed"; exit 1; }' % (self.bpn, self.bpn))
800
801 # Self contained devtool deploy-target
802 cmd_lines.append(self.gen_deploy_target_script(args))
803
804 return self.write_script(cmd_lines, 'install_and_deploy')
805
806 def write_script(self, cmd_lines, script_name):
807 bb.utils.mkdirhier(self.ide_sdk_scripts_dir)
808 script_name_arch = script_name + '_' + self.recipe_id
809 script_file = os.path.join(self.ide_sdk_scripts_dir, script_name_arch)
810 with open(script_file, 'w') as script_f:
811 script_f.write(os.linesep.join(cmd_lines))
812 st = os.stat(script_file)
813 os.chmod(script_file, st.st_mode | stat.S_IEXEC)
814 return script_file
815
816 @property
817 def oe_init_build_env(self):
818 """Find the oe-init-build-env used for this setup"""
819 oe_init_dir = self.oe_init_dir
820 if oe_init_dir:
821 return os.path.join(oe_init_dir, RecipeModified.OE_INIT_BUILD_ENV)
822 return None
823
824 @property
825 def oe_init_dir(self):
826 """Find the directory where the oe-init-build-env is located
827
828 Assumption: There might be a layer with higher priority than poky
829 which provides to oe-init-build-env in the layer's toplevel folder.
830 """
831 if not self.__oe_init_dir:
832 for layer in reversed(self.bblayers):
833 result = subprocess.run(
834 ['git', 'rev-parse', '--show-toplevel'], cwd=layer, capture_output=True)
835 if result.returncode == 0:
836 oe_init_dir = result.stdout.decode('utf-8').strip()
837 oe_init_path = os.path.join(
838 oe_init_dir, RecipeModified.OE_INIT_BUILD_ENV)
839 if os.path.exists(oe_init_path):
840 logger.debug("Using %s from: %s" % (
841 RecipeModified.OE_INIT_BUILD_ENV, oe_init_path))
842 self.__oe_init_dir = oe_init_dir
843 break
844 if not self.__oe_init_dir:
845 logger.error("Cannot find the bitbake top level folder")
846 return self.__oe_init_dir
847
848
849def ide_setup(args, config, basepath, workspace):
850 """Generate the IDE configuration for the workspace"""
851
852 # Explicitely passing some special recipes does not make sense
853 for recipe in args.recipenames:
854 if recipe in ['meta-ide-support', 'build-sysroots']:
855 raise DevtoolError("Invalid recipe: %s." % recipe)
856
857 # Collect information about tasks which need to be bitbaked
858 bootstrap_tasks = []
859 bootstrap_tasks_late = []
860 tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
861 try:
862 # define mode depending on recipes which need to be processed
863 recipes_image_names = []
864 recipes_modified_names = []
865 recipes_other_names = []
866 for recipe in args.recipenames:
867 try:
868 check_workspace_recipe(
869 workspace, recipe, bbclassextend=True)
870 recipes_modified_names.append(recipe)
871 except DevtoolError:
872 recipe_d = parse_recipe(
873 config, tinfoil, recipe, appends=True, filter_workspace=False)
874 if not recipe_d:
875 raise DevtoolError("Parsing recipe %s failed" % recipe)
876 if bb.data.inherits_class('image', recipe_d):
877 recipes_image_names.append(recipe)
878 else:
879 recipes_other_names.append(recipe)
880
881 invalid_params = False
882 if args.mode == DevtoolIdeMode.shared:
883 if len(recipes_modified_names):
884 logger.error("In shared sysroots mode modified recipes %s cannot be handled." % str(
885 recipes_modified_names))
886 invalid_params = True
887 if args.mode == DevtoolIdeMode.modified:
888 if len(recipes_other_names):
889 logger.error("Only in shared sysroots mode not modified recipes %s can be handled." % str(
890 recipes_other_names))
891 invalid_params = True
892 if len(recipes_image_names) != 1:
893 logger.error(
894 "One image recipe is required as the rootfs for the remote development.")
895 invalid_params = True
896 for modified_recipe_name in recipes_modified_names:
897 if modified_recipe_name.startswith('nativesdk-') or modified_recipe_name.endswith('-native'):
898 logger.error(
899 "Only cross compiled recipes are support. %s is not cross." % modified_recipe_name)
900 invalid_params = True
901
902 if invalid_params:
903 raise DevtoolError("Invalid parameters are passed.")
904
905 # For the shared sysroots mode, add all dependencies of all the images to the sysroots
906 # For the modified mode provide one rootfs and the corresponding debug symbols via rootfs-dbg
907 recipes_images = []
908 for recipes_image_name in recipes_image_names:
909 logger.info("Using image: %s" % recipes_image_name)
910 recipe_image = RecipeImage(recipes_image_name)
911 recipe_image.initialize(config, tinfoil)
912 bootstrap_tasks += recipe_image.bootstrap_tasks
913 recipes_images.append(recipe_image)
914
915 # Provide a Direct SDK with shared sysroots
916 recipes_not_modified = []
917 if args.mode == DevtoolIdeMode.shared:
918 ide_support = RecipeMetaIdeSupport()
919 ide_support.initialize(config, tinfoil)
920 bootstrap_tasks += ide_support.bootstrap_tasks
921
922 logger.info("Adding %s to the Direct SDK sysroots." %
923 str(recipes_other_names))
924 for recipe_name in recipes_other_names:
925 recipe_not_modified = RecipeNotModified(recipe_name)
926 bootstrap_tasks += recipe_not_modified.bootstrap_tasks
927 recipes_not_modified.append(recipe_not_modified)
928
929 build_sysroots = RecipeBuildSysroots()
930 build_sysroots.initialize(config, tinfoil)
931 bootstrap_tasks_late += build_sysroots.bootstrap_tasks
932 shared_env = SharedSysrootsEnv()
933 shared_env.initialize(ide_support, build_sysroots)
934
935 recipes_modified = []
936 if args.mode == DevtoolIdeMode.modified:
937 logger.info("Setting up workspaces for modified recipe: %s" %
938 str(recipes_modified_names))
939 gdbs_cross = {}
940 for recipe_name in recipes_modified_names:
941 recipe_modified = RecipeModified(recipe_name)
942 recipe_modified.initialize(config, workspace, tinfoil)
943 bootstrap_tasks += recipe_modified.bootstrap_tasks
944 recipes_modified.append(recipe_modified)
945
946 if recipe_modified.target_arch not in gdbs_cross:
947 target_device = TargetDevice(args)
948 gdb_cross = RecipeGdbCross(
949 args, recipe_modified.target_arch, target_device)
950 gdb_cross.initialize(config, workspace, tinfoil)
951 bootstrap_tasks += gdb_cross.bootstrap_tasks
952 gdbs_cross[recipe_modified.target_arch] = gdb_cross
953 recipe_modified.gdb_cross = gdbs_cross[recipe_modified.target_arch]
954
955 finally:
956 tinfoil.shutdown()
957
958 if not args.skip_bitbake:
959 bb_cmd = 'bitbake '
960 if args.bitbake_k:
961 bb_cmd += "-k "
962 bb_cmd_early = bb_cmd + ' '.join(bootstrap_tasks)
963 exec_build_env_command(
964 config.init_path, basepath, bb_cmd_early, watch=True)
965 if bootstrap_tasks_late:
966 bb_cmd_late = bb_cmd + ' '.join(bootstrap_tasks_late)
967 exec_build_env_command(
968 config.init_path, basepath, bb_cmd_late, watch=True)
969
970 for recipe_image in recipes_images:
971 if (recipe_image.gdbserver_missing):
972 logger.warning(
973 "gdbserver not installed in image %s. Remote debugging will not be available" % recipe_image)
974
975 if recipe_image.combine_dbg_image is False:
976 logger.warning(
977 'IMAGE_CLASSES += "image-combined-dbg" is missing for image %s. Remote debugging will not find debug symbols from rootfs-dbg.' % recipe_image)
978
979 # Instantiate the active IDE plugin
980 ide = ide_plugins[args.ide]()
981 if args.mode == DevtoolIdeMode.shared:
982 ide.setup_shared_sysroots(shared_env)
983 elif args.mode == DevtoolIdeMode.modified:
984 for recipe_modified in recipes_modified:
985 if recipe_modified.build_tool is BuildTool.CMAKE:
986 recipe_modified.cmake_preset()
987 if recipe_modified.build_tool is BuildTool.MESON:
988 recipe_modified.gen_meson_wrapper()
989 ide.setup_modified_recipe(
990 args, recipe_image, recipe_modified)
991 else:
992 raise DevtoolError("Must not end up here.")
993
994
995def register_commands(subparsers, context):
996 """Register devtool subcommands from this plugin"""
997
998 global ide_plugins
999
1000 # Search for IDE plugins in all sub-folders named ide_plugins where devtool seraches for plugins.
1001 pluginpaths = [os.path.join(path, 'ide_plugins')
1002 for path in context.pluginpaths]
1003 ide_plugin_modules = []
1004 for pluginpath in pluginpaths:
1005 scriptutils.load_plugins(logger, ide_plugin_modules, pluginpath)
1006
1007 for ide_plugin_module in ide_plugin_modules:
1008 if hasattr(ide_plugin_module, 'register_ide_plugin'):
1009 ide_plugin_module.register_ide_plugin(ide_plugins)
1010 # Sort plugins according to their priority. The first entry is the default IDE plugin.
1011 ide_plugins = dict(sorted(ide_plugins.items(),
1012 key=lambda p: p[1].ide_plugin_priority(), reverse=True))
1013
1014 parser_ide_sdk = subparsers.add_parser('ide-sdk', group='working', order=50, formatter_class=RawTextHelpFormatter,
1015 help='Setup the SDK and configure the IDE')
1016 parser_ide_sdk.add_argument(
1017 'recipenames', nargs='+', help='Generate an IDE configuration suitable to work on the given recipes.\n'
1018 'Depending on the --mode paramter different types of SDKs and IDE configurations are generated.')
1019 parser_ide_sdk.add_argument(
1020 '-m', '--mode', type=DevtoolIdeMode, default=DevtoolIdeMode.modified,
1021 help='Different SDK types are supported:\n'
1022 '- "' + DevtoolIdeMode.modified.name + '" (default):\n'
1023 ' devtool modify creates a workspace to work on the source code of a recipe.\n'
1024 ' devtool ide-sdk builds the SDK and generates the IDE configuration(s) in the workspace directorie(s)\n'
1025 ' Usage example:\n'
1026 ' devtool modify cmake-example\n'
1027 ' devtool ide-sdk cmake-example core-image-minimal\n'
1028 ' Start the IDE in the workspace folder\n'
1029 ' At least one devtool modified recipe plus one image recipe are required:\n'
1030 ' The image recipe is used to generate the target image and the remote debug configuration.\n'
1031 '- "' + DevtoolIdeMode.shared.name + '":\n'
1032 ' Usage example:\n'
1033 ' devtool ide-sdk -m ' + DevtoolIdeMode.shared.name + ' recipe(s)\n'
1034 ' This command generates a cross-toolchain as well as the corresponding shared sysroot directories.\n'
1035 ' To use this tool-chain the environment-* file found in the deploy..image folder needs to be sourced into a shell.\n'
1036 ' In case of VSCode and cmake the tool-chain is also exposed as a cmake-kit')
1037 default_ide = list(ide_plugins.keys())[0]
1038 parser_ide_sdk.add_argument(
1039 '-i', '--ide', choices=ide_plugins.keys(), default=default_ide,
1040 help='Setup the configuration for this IDE (default: %s)' % default_ide)
1041 parser_ide_sdk.add_argument(
1042 '-t', '--target', default='root@192.168.7.2',
1043 help='Live target machine running an ssh server: user@hostname.')
1044 parser_ide_sdk.add_argument(
1045 '-G', '--gdbserver-port-start', default="1234", help='port where gdbserver is listening.')
1046 parser_ide_sdk.add_argument(
1047 '-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
1048 parser_ide_sdk.add_argument(
1049 '-e', '--ssh-exec', help='Executable to use in place of ssh')
1050 parser_ide_sdk.add_argument(
1051 '-P', '--port', help='Specify ssh port to use for connection to the target')
1052 parser_ide_sdk.add_argument(
1053 '-I', '--key', help='Specify ssh private key for connection to the target')
1054 parser_ide_sdk.add_argument(
1055 '--skip-bitbake', help='Generate IDE configuration but skip calling bibtake to update the SDK.', action='store_true')
1056 parser_ide_sdk.add_argument(
1057 '-k', '--bitbake-k', help='Pass -k parameter to bitbake', action='store_true')
1058 parser_ide_sdk.add_argument(
1059 '--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false')
1060 parser_ide_sdk.add_argument(
1061 '-n', '--dry-run', help='List files to be undeployed only', action='store_true')
1062 parser_ide_sdk.add_argument(
1063 '-s', '--show-status', help='Show progress/status output', action='store_true')
1064 parser_ide_sdk.add_argument(
1065 '-p', '--no-preserve', help='Do not preserve existing files', action='store_true')
1066 parser_ide_sdk.add_argument(
1067 '--no-check-space', help='Do not check for available space before deploying', action='store_true')
1068 parser_ide_sdk.add_argument(
1069 '--debug-build-config', help='Use debug build flags, for example set CMAKE_BUILD_TYPE=Debug', action='store_true')
1070 parser_ide_sdk.set_defaults(func=ide_setup)
diff --git a/scripts/lib/devtool/menuconfig.py b/scripts/lib/devtool/menuconfig.py
index 95384c5333..18daef30c3 100644
--- a/scripts/lib/devtool/menuconfig.py
+++ b/scripts/lib/devtool/menuconfig.py
@@ -3,6 +3,8 @@
3# Copyright (C) 2018 Xilinx 3# Copyright (C) 2018 Xilinx
4# Written by: Chandana Kalluri <ckalluri@xilinx.com> 4# Written by: Chandana Kalluri <ckalluri@xilinx.com>
5# 5#
6# SPDX-License-Identifier: MIT
7#
6# This program is free software; you can redistribute it and/or modify 8# This program is free software; you can redistribute it and/or modify
7# it under the terms of the GNU General Public License version 2 as 9# it under the terms of the GNU General Public License version 2 as
8# published by the Free Software Foundation. 10# published by the Free Software Foundation.
@@ -43,7 +45,7 @@ def menuconfig(args, config, basepath, workspace):
43 return 1 45 return 1
44 46
45 check_workspace_recipe(workspace, args.component) 47 check_workspace_recipe(workspace, args.component)
46 pn = rd.getVar('PN', True) 48 pn = rd.getVar('PN')
47 49
48 if not rd.getVarFlag('do_menuconfig','task'): 50 if not rd.getVarFlag('do_menuconfig','task'):
49 raise DevtoolError("This recipe does not support menuconfig option") 51 raise DevtoolError("This recipe does not support menuconfig option")
diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py
index 3aa42a1466..9aefd7e354 100644
--- a/scripts/lib/devtool/sdk.py
+++ b/scripts/lib/devtool/sdk.py
@@ -207,7 +207,7 @@ def sdk_update(args, config, basepath, workspace):
207 if not sstate_mirrors: 207 if not sstate_mirrors:
208 with open(os.path.join(conf_dir, 'site.conf'), 'a') as f: 208 with open(os.path.join(conf_dir, 'site.conf'), 'a') as f:
209 f.write('SCONF_VERSION = "%s"\n' % site_conf_version) 209 f.write('SCONF_VERSION = "%s"\n' % site_conf_version)
210 f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver) 210 f.write('SSTATE_MIRRORS:append = " file://.* %s/sstate-cache/PATH"\n' % updateserver)
211 finally: 211 finally:
212 shutil.rmtree(tmpsdk_dir) 212 shutil.rmtree(tmpsdk_dir)
213 213
@@ -300,7 +300,8 @@ def sdk_install(args, config, basepath, workspace):
300 return 2 300 return 2
301 301
302 try: 302 try:
303 exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots', watch=True) 303 exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots -c build_native_sysroot', watch=True)
304 exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots -c build_target_sysroot', watch=True)
304 except bb.process.ExecutionError as e: 305 except bb.process.ExecutionError as e:
305 raise DevtoolError('Failed to bitbake build-sysroots:\n%s' % (str(e))) 306 raise DevtoolError('Failed to bitbake build-sysroots:\n%s' % (str(e)))
306 307
diff --git a/scripts/lib/devtool/search.py b/scripts/lib/devtool/search.py
index d24040df37..70b81cac5e 100644
--- a/scripts/lib/devtool/search.py
+++ b/scripts/lib/devtool/search.py
@@ -62,10 +62,11 @@ def search(args, config, basepath, workspace):
62 with open(os.path.join(pkgdata_dir, 'runtime', pkg), 'r') as f: 62 with open(os.path.join(pkgdata_dir, 'runtime', pkg), 'r') as f:
63 for line in f: 63 for line in f:
64 if ': ' in line: 64 if ': ' in line:
65 splitline = line.split(':', 1) 65 splitline = line.split(': ', 1)
66 key = splitline[0] 66 key = splitline[0]
67 value = splitline[1].strip() 67 value = splitline[1].strip()
68 if key in ['PKG_%s' % pkg, 'DESCRIPTION', 'FILES_INFO'] or key.startswith('FILERPROVIDES_'): 68 key = key.replace(":" + pkg, "")
69 if key in ['PKG', 'DESCRIPTION', 'FILES_INFO', 'FILERPROVIDES']:
69 if keyword_rc.search(value): 70 if keyword_rc.search(value):
70 match = True 71 match = True
71 break 72 break
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 7b62b7e7b8..bd009f44b1 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -147,6 +147,8 @@ def add(args, config, basepath, workspace):
147 extracmdopts += ' -a' 147 extracmdopts += ' -a'
148 if args.npm_dev: 148 if args.npm_dev:
149 extracmdopts += ' --npm-dev' 149 extracmdopts += ' --npm-dev'
150 if args.no_pypi:
151 extracmdopts += ' --no-pypi'
150 if args.mirrors: 152 if args.mirrors:
151 extracmdopts += ' --mirrors' 153 extracmdopts += ' --mirrors'
152 if args.srcrev: 154 if args.srcrev:
@@ -234,10 +236,14 @@ def add(args, config, basepath, workspace):
234 if args.fetchuri and not args.no_git: 236 if args.fetchuri and not args.no_git:
235 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data) 237 setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data)
236 238
237 initial_rev = None 239 initial_rev = {}
238 if os.path.exists(os.path.join(srctree, '.git')): 240 if os.path.exists(os.path.join(srctree, '.git')):
239 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 241 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
240 initial_rev = stdout.rstrip() 242 initial_rev["."] = stdout.rstrip()
243 (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse HEAD` $PWD\'', cwd=srctree)
244 for line in stdout.splitlines():
245 (rev, submodule) = line.split()
246 initial_rev[os.path.relpath(submodule, srctree)] = rev
241 247
242 if args.src_subdir: 248 if args.src_subdir:
243 srctree = os.path.join(srctree, args.src_subdir) 249 srctree = os.path.join(srctree, args.src_subdir)
@@ -251,16 +257,17 @@ def add(args, config, basepath, workspace):
251 if b_is_s: 257 if b_is_s:
252 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree) 258 f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree)
253 if initial_rev: 259 if initial_rev:
254 f.write('\n# initial_rev: %s\n' % initial_rev) 260 for key, value in initial_rev.items():
261 f.write('\n# initial_rev %s: %s\n' % (key, value))
255 262
256 if args.binary: 263 if args.binary:
257 f.write('do_install_append() {\n') 264 f.write('do_install:append() {\n')
258 f.write(' rm -rf ${D}/.git\n') 265 f.write(' rm -rf ${D}/.git\n')
259 f.write(' rm -f ${D}/singletask.lock\n') 266 f.write(' rm -f ${D}/singletask.lock\n')
260 f.write('}\n') 267 f.write('}\n')
261 268
262 if bb.data.inherits_class('npm', rd): 269 if bb.data.inherits_class('npm', rd):
263 f.write('python do_configure_append() {\n') 270 f.write('python do_configure:append() {\n')
264 f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n') 271 f.write(' pkgdir = d.getVar("NPM_PACKAGE")\n')
265 f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n') 272 f.write(' lockfile = os.path.join(pkgdir, "singletask.lock")\n')
266 f.write(' bb.utils.remove(lockfile)\n') 273 f.write(' bb.utils.remove(lockfile)\n')
@@ -318,10 +325,6 @@ def _check_compatible_recipe(pn, d):
318 raise DevtoolError("The %s recipe is a packagegroup, and therefore is " 325 raise DevtoolError("The %s recipe is a packagegroup, and therefore is "
319 "not supported by this tool" % pn, 4) 326 "not supported by this tool" % pn, 4)
320 327
321 if bb.data.inherits_class('meta', d):
322 raise DevtoolError("The %s recipe is a meta-recipe, and therefore is "
323 "not supported by this tool" % pn, 4)
324
325 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'): 328 if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'):
326 # Not an incompatibility error per se, so we don't pass the error code 329 # Not an incompatibility error per se, so we don't pass the error code
327 raise DevtoolError("externalsrc is currently enabled for the %s " 330 raise DevtoolError("externalsrc is currently enabled for the %s "
@@ -357,7 +360,7 @@ def _move_file(src, dst, dry_run_outdir=None, base_outdir=None):
357 bb.utils.mkdirhier(dst_d) 360 bb.utils.mkdirhier(dst_d)
358 shutil.move(src, dst) 361 shutil.move(src, dst)
359 362
360def _copy_file(src, dst, dry_run_outdir=None): 363def _copy_file(src, dst, dry_run_outdir=None, base_outdir=None):
361 """Copy a file. Creates all the directory components of destination path.""" 364 """Copy a file. Creates all the directory components of destination path."""
362 dry_run_suffix = ' (dry-run)' if dry_run_outdir else '' 365 dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
363 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix)) 366 logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix))
@@ -457,7 +460,7 @@ def sync(args, config, basepath, workspace):
457 finally: 460 finally:
458 tinfoil.shutdown() 461 tinfoil.shutdown()
459 462
460def symlink_oelocal_files_srctree(rd,srctree): 463def symlink_oelocal_files_srctree(rd, srctree):
461 import oe.patch 464 import oe.patch
462 if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): 465 if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')):
463 # If recipe extracts to ${WORKDIR}, symlink the files into the srctree 466 # If recipe extracts to ${WORKDIR}, symlink the files into the srctree
@@ -481,11 +484,7 @@ def symlink_oelocal_files_srctree(rd,srctree):
481 os.symlink('oe-local-files/%s' % fn, destpth) 484 os.symlink('oe-local-files/%s' % fn, destpth)
482 addfiles.append(os.path.join(relpth, fn)) 485 addfiles.append(os.path.join(relpth, fn))
483 if addfiles: 486 if addfiles:
484 bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree) 487 oe.patch.GitApplyTree.commitIgnored("Add local file symlinks", dir=srctree, files=addfiles, d=rd)
485 useroptions = []
486 oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd)
487 bb.process.run('git %s commit -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree)
488
489 488
490def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): 489def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False):
491 """Extract sources of a recipe""" 490 """Extract sources of a recipe"""
@@ -523,8 +522,10 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
523 history = d.varhistory.variable('SRC_URI') 522 history = d.varhistory.variable('SRC_URI')
524 for event in history: 523 for event in history:
525 if not 'flag' in event: 524 if not 'flag' in event:
526 if event['op'].startswith(('_append[', '_prepend[')): 525 if event['op'].startswith((':append[', ':prepend[')):
527 extra_overrides.append(event['op'].split('[')[1].split(']')[0]) 526 override = event['op'].split('[')[1].split(']')[0]
527 if not override.startswith('pn-'):
528 extra_overrides.append(override)
528 # We want to remove duplicate overrides. If a recipe had multiple 529 # We want to remove duplicate overrides. If a recipe had multiple
529 # SRC_URI_override += values it would cause mulitple instances of 530 # SRC_URI_override += values it would cause mulitple instances of
530 # overrides. This doesn't play nicely with things like creating a 531 # overrides. This doesn't play nicely with things like creating a
@@ -569,6 +570,9 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
569 logger.debug('writing append file %s' % appendfile) 570 logger.debug('writing append file %s' % appendfile)
570 with open(appendfile, 'a') as f: 571 with open(appendfile, 'a') as f:
571 f.write('###--- _extract_source\n') 572 f.write('###--- _extract_source\n')
573 f.write('deltask do_recipe_qa\n')
574 f.write('deltask do_recipe_qa_setscene\n')
575 f.write('ERROR_QA:remove = "patch-fuzz"\n')
572 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir) 576 f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir)
573 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch) 577 f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch)
574 if not is_kernel_yocto: 578 if not is_kernel_yocto:
@@ -586,6 +590,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
586 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps') 590 preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps')
587 with open(preservestampfile, 'w') as f: 591 with open(preservestampfile, 'w') as f:
588 f.write(d.getVar('STAMP')) 592 f.write(d.getVar('STAMP'))
593 tinfoil.modified_files()
589 try: 594 try:
590 if is_kernel_yocto: 595 if is_kernel_yocto:
591 # We need to generate the kernel config 596 # We need to generate the kernel config
@@ -648,23 +653,34 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
648 653
649 if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer): 654 if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer):
650 shutil.rmtree(workshareddir) 655 shutil.rmtree(workshareddir)
651 oe.path.copyhardlinktree(srcsubdir,workshareddir) 656 oe.path.copyhardlinktree(srcsubdir, workshareddir)
652 elif not os.path.exists(workshareddir): 657 elif not os.path.exists(workshareddir):
653 oe.path.copyhardlinktree(srcsubdir,workshareddir) 658 oe.path.copyhardlinktree(srcsubdir, workshareddir)
654 659
655 tempdir_localdir = os.path.join(tempdir, 'oe-local-files') 660 tempdir_localdir = os.path.join(tempdir, 'oe-local-files')
656 srctree_localdir = os.path.join(srctree, 'oe-local-files') 661 srctree_localdir = os.path.join(srctree, 'oe-local-files')
657 662
658 if sync: 663 if sync:
659 bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) 664 try:
660 665 logger.info('Backing up current %s branch as branch: %s.bak' % (devbranch, devbranch))
661 # Move oe-local-files directory to srctree 666 bb.process.run('git branch -f ' + devbranch + '.bak', cwd=srctree)
662 # As the oe-local-files is not part of the constructed git tree, 667
663 # remove them directly during the synchrounizating might surprise 668 # Use git fetch to update the source with the current recipe
664 # the users. Instead, we move it to oe-local-files.bak and remind 669 # To be able to update the currently checked out branch with
665 # user in the log message. 670 # possibly new history (no fast-forward) git needs to be told
671 # that's ok
672 logger.info('Syncing source files including patches to git branch: %s' % devbranch)
673 bb.process.run('git fetch --update-head-ok --force file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree)
674 except bb.process.ExecutionError as e:
675 raise DevtoolError("Error when syncing source files to local checkout: %s" % str(e))
676
677 # Move the oe-local-files directory to srctree.
678 # As oe-local-files is not part of the constructed git tree,
679 # removing it directly during the synchronization might surprise
680 # the user. Instead, we move it to oe-local-files.bak and remind
681 # the user in the log message.
666 if os.path.exists(srctree_localdir + '.bak'): 682 if os.path.exists(srctree_localdir + '.bak'):
667 shutil.rmtree(srctree_localdir, srctree_localdir + '.bak') 683 shutil.rmtree(srctree_localdir + '.bak')
668 684
669 if os.path.exists(srctree_localdir): 685 if os.path.exists(srctree_localdir):
670 logger.info('Backing up current local file directory %s' % srctree_localdir) 686 logger.info('Backing up current local file directory %s' % srctree_localdir)
@@ -680,7 +696,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
680 shutil.move(tempdir_localdir, srcsubdir) 696 shutil.move(tempdir_localdir, srcsubdir)
681 697
682 shutil.move(srcsubdir, srctree) 698 shutil.move(srcsubdir, srctree)
683 symlink_oelocal_files_srctree(d,srctree) 699 symlink_oelocal_files_srctree(d, srctree)
684 700
685 if is_kernel_yocto: 701 if is_kernel_yocto:
686 logger.info('Copying kernel config to srctree') 702 logger.info('Copying kernel config to srctree')
@@ -746,14 +762,14 @@ def _check_preserve(config, recipename):
746 os.remove(removefile) 762 os.remove(removefile)
747 else: 763 else:
748 tf.write(line) 764 tf.write(line)
749 os.rename(newfile, origfile) 765 bb.utils.rename(newfile, origfile)
750 766
751def get_staging_kver(srcdir): 767def get_staging_kver(srcdir):
752 # Kernel version from work-shared 768 # Kernel version from work-shared
753 kerver = [] 769 kerver = []
754 staging_kerVer="" 770 staging_kerVer=""
755 if os.path.exists(srcdir) and os.listdir(srcdir): 771 if os.path.exists(srcdir) and os.listdir(srcdir):
756 with open(os.path.join(srcdir,"Makefile")) as f: 772 with open(os.path.join(srcdir, "Makefile")) as f:
757 version = [next(f) for x in range(5)][1:4] 773 version = [next(f) for x in range(5)][1:4]
758 for word in version: 774 for word in version:
759 kerver.append(word.split('= ')[1].split('\n')[0]) 775 kerver.append(word.split('= ')[1].split('\n')[0])
@@ -763,10 +779,20 @@ def get_staging_kver(srcdir):
763def get_staging_kbranch(srcdir): 779def get_staging_kbranch(srcdir):
764 staging_kbranch = "" 780 staging_kbranch = ""
765 if os.path.exists(srcdir) and os.listdir(srcdir): 781 if os.path.exists(srcdir) and os.listdir(srcdir):
766 (branch, _) = bb.process.run('git branch | grep \* | cut -d \' \' -f2', cwd=srcdir) 782 (branch, _) = bb.process.run('git branch | grep \\* | cut -d \' \' -f2', cwd=srcdir)
767 staging_kbranch = "".join(branch.split('\n')[0]) 783 staging_kbranch = "".join(branch.split('\n')[0])
768 return staging_kbranch 784 return staging_kbranch
769 785
786def get_real_srctree(srctree, s, workdir):
787 # Check that recipe isn't using a shared workdir
788 s = os.path.abspath(s)
789 workdir = os.path.abspath(workdir)
790 if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir:
791 # Handle if S is set to a subdirectory of the source
792 srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1]
793 srctree = os.path.join(srctree, srcsubdir)
794 return srctree
795
770def modify(args, config, basepath, workspace): 796def modify(args, config, basepath, workspace):
771 """Entry point for the devtool 'modify' subcommand""" 797 """Entry point for the devtool 'modify' subcommand"""
772 import bb 798 import bb
@@ -811,8 +837,8 @@ def modify(args, config, basepath, workspace):
811 837
812 _check_compatible_recipe(pn, rd) 838 _check_compatible_recipe(pn, rd)
813 839
814 initial_rev = None 840 initial_revs = {}
815 commits = [] 841 commits = {}
816 check_commits = False 842 check_commits = False
817 843
818 if bb.data.inherits_class('kernel-yocto', rd): 844 if bb.data.inherits_class('kernel-yocto', rd):
@@ -824,10 +850,10 @@ def modify(args, config, basepath, workspace):
824 staging_kerVer = get_staging_kver(srcdir) 850 staging_kerVer = get_staging_kver(srcdir)
825 staging_kbranch = get_staging_kbranch(srcdir) 851 staging_kbranch = get_staging_kbranch(srcdir)
826 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): 852 if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch):
827 oe.path.copyhardlinktree(srcdir,srctree) 853 oe.path.copyhardlinktree(srcdir, srctree)
828 workdir = rd.getVar('WORKDIR') 854 workdir = rd.getVar('WORKDIR')
829 srcsubdir = rd.getVar('S') 855 srcsubdir = rd.getVar('S')
830 localfilesdir = os.path.join(srctree,'oe-local-files') 856 localfilesdir = os.path.join(srctree, 'oe-local-files')
831 # Move local source files into separate subdir 857 # Move local source files into separate subdir
832 recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] 858 recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)]
833 local_files = oe.recipeutils.get_recipe_local_files(rd) 859 local_files = oe.recipeutils.get_recipe_local_files(rd)
@@ -851,9 +877,9 @@ def modify(args, config, basepath, workspace):
851 for fname in local_files: 877 for fname in local_files:
852 _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) 878 _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname))
853 with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: 879 with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f:
854 f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n') 880 f.write('# Ignore local files, by default. Remove this file if you want to commit the directory to Git\n*\n')
855 881
856 symlink_oelocal_files_srctree(rd,srctree) 882 symlink_oelocal_files_srctree(rd, srctree)
857 883
858 task = 'do_configure' 884 task = 'do_configure'
859 res = tinfoil.build_targets(pn, task, handle_events=True) 885 res = tinfoil.build_targets(pn, task, handle_events=True)
@@ -861,22 +887,30 @@ def modify(args, config, basepath, workspace):
861 # Copy .config to workspace 887 # Copy .config to workspace
862 kconfpath = rd.getVar('B') 888 kconfpath = rd.getVar('B')
863 logger.info('Copying kernel config to workspace') 889 logger.info('Copying kernel config to workspace')
864 shutil.copy2(os.path.join(kconfpath, '.config'),srctree) 890 shutil.copy2(os.path.join(kconfpath, '.config'), srctree)
865 891
866 # Set this to true, we still need to get initial_rev 892 # Set this to true, we still need to get initial_rev
867 # by parsing the git repo 893 # by parsing the git repo
868 args.no_extract = True 894 args.no_extract = True
869 895
870 if not args.no_extract: 896 if not args.no_extract:
871 initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 897 initial_revs["."], _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
872 if not initial_rev: 898 if not initial_revs["."]:
873 return 1 899 return 1
874 logger.info('Source tree extracted to %s' % srctree) 900 logger.info('Source tree extracted to %s' % srctree)
901
875 if os.path.exists(os.path.join(srctree, '.git')): 902 if os.path.exists(os.path.join(srctree, '.git')):
876 # Get list of commits since this revision 903 # Get list of commits since this revision
877 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree) 904 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_revs["."], cwd=srctree)
878 commits = stdout.split() 905 commits["."] = stdout.split()
879 check_commits = True 906 check_commits = True
907 (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree)
908 for line in stdout.splitlines():
909 (rev, submodule_path) = line.split()
910 submodule = os.path.relpath(submodule_path, srctree)
911 initial_revs[submodule] = rev
912 (stdout, _) = bb.process.run('git rev-list --reverse devtool-base..HEAD', cwd=submodule_path)
913 commits[submodule] = stdout.split()
880 else: 914 else:
881 if os.path.exists(os.path.join(srctree, '.git')): 915 if os.path.exists(os.path.join(srctree, '.git')):
882 # Check if it's a tree previously extracted by us. This is done 916 # Check if it's a tree previously extracted by us. This is done
@@ -893,11 +927,11 @@ def modify(args, config, basepath, workspace):
893 for line in stdout.splitlines(): 927 for line in stdout.splitlines():
894 if line.startswith('*'): 928 if line.startswith('*'):
895 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree) 929 (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree)
896 initial_rev = stdout.rstrip() 930 initial_revs["."] = stdout.rstrip()
897 if not initial_rev: 931 if "." not in initial_revs:
898 # Otherwise, just grab the head revision 932 # Otherwise, just grab the head revision
899 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) 933 (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
900 initial_rev = stdout.rstrip() 934 initial_revs["."] = stdout.rstrip()
901 935
902 branch_patches = {} 936 branch_patches = {}
903 if check_commits: 937 if check_commits:
@@ -914,62 +948,81 @@ def modify(args, config, basepath, workspace):
914 seen_patches = [] 948 seen_patches = []
915 for branch in branches: 949 for branch in branches:
916 branch_patches[branch] = [] 950 branch_patches[branch] = []
917 (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree) 951 (stdout, _) = bb.process.run('git rev-list devtool-base..%s' % branch, cwd=srctree)
918 for line in stdout.splitlines(): 952 for sha1 in stdout.splitlines():
919 line = line.strip() 953 notes = oe.patch.GitApplyTree.getNotes(srctree, sha1.strip())
920 if line.startswith(oe.patch.GitApplyTree.patch_line_prefix): 954 origpatch = notes.get(oe.patch.GitApplyTree.original_patch)
921 origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip() 955 if origpatch and origpatch not in seen_patches:
922 if not origpatch in seen_patches: 956 seen_patches.append(origpatch)
923 seen_patches.append(origpatch) 957 branch_patches[branch].append(origpatch)
924 branch_patches[branch].append(origpatch)
925 958
926 # Need to grab this here in case the source is within a subdirectory 959 # Need to grab this here in case the source is within a subdirectory
927 srctreebase = srctree 960 srctreebase = srctree
928 961 srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR'))
929 # Check that recipe isn't using a shared workdir
930 s = os.path.abspath(rd.getVar('S'))
931 workdir = os.path.abspath(rd.getVar('WORKDIR'))
932 if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir:
933 # Handle if S is set to a subdirectory of the source
934 srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1]
935 srctree = os.path.join(srctree, srcsubdir)
936 962
937 bb.utils.mkdirhier(os.path.dirname(appendfile)) 963 bb.utils.mkdirhier(os.path.dirname(appendfile))
938 with open(appendfile, 'w') as f: 964 with open(appendfile, 'w') as f:
939 f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n') 965 # if not present, add type=git-dependency to the secondary sources
966 # (non local files) so they can be extracted correctly when building a recipe after
967 # doing a devtool modify on it
968 src_uri = rd.getVar('SRC_URI').split()
969 src_uri_append = []
970 src_uri_remove = []
971
972 # Assume first entry is main source extracted in ${S} so skip it
973 src_uri = src_uri[1::]
974
975 # Add "type=git-dependency" to all non local sources
976 for url in src_uri:
977 if not url.startswith('file://') and not 'type=' in url:
978 src_uri_remove.append(url)
979 src_uri_append.append('%s;type=git-dependency' % url)
980
981 if src_uri_remove:
982 f.write('SRC_URI:remove = "%s"\n' % ' '.join(src_uri_remove))
983 f.write('SRC_URI:append = " %s"\n\n' % ' '.join(src_uri_append))
984
985 f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n')
940 # Local files can be modified/tracked in separate subdir under srctree 986 # Local files can be modified/tracked in separate subdir under srctree
941 # Mostly useful for packages with S != WORKDIR 987 # Mostly useful for packages with S != WORKDIR
942 f.write('FILESPATH_prepend := "%s:"\n' % 988 f.write('FILESPATH:prepend := "%s:"\n' %
943 os.path.join(srctreebase, 'oe-local-files')) 989 os.path.join(srctreebase, 'oe-local-files'))
944 f.write('# srctreebase: %s\n' % srctreebase) 990 f.write('# srctreebase: %s\n' % srctreebase)
945 991
946 f.write('\ninherit externalsrc\n') 992 f.write('\ninherit externalsrc\n')
947 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n') 993 f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n')
948 f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree)) 994 f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree))
949 995
950 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd) 996 b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd)
951 if b_is_s: 997 if b_is_s:
952 f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree)) 998 f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree))
953 999
954 if bb.data.inherits_class('kernel', rd): 1000 if bb.data.inherits_class('kernel', rd):
955 f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout ' 1001 f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout '
956 'do_fetch do_unpack do_kernel_configme do_kernel_configcheck"\n') 1002 'do_fetch do_unpack do_kernel_configcheck"\n')
957 f.write('\ndo_patch[noexec] = "1"\n') 1003 f.write('\ndo_patch[noexec] = "1"\n')
958 f.write('\ndo_configure_append() {\n' 1004 f.write('\ndo_configure:append() {\n'
959 ' cp ${B}/.config ${S}/.config.baseline\n' 1005 ' cp ${B}/.config ${S}/.config.baseline\n'
960 ' ln -sfT ${B}/.config ${S}/.config.new\n' 1006 ' ln -sfT ${B}/.config ${S}/.config.new\n'
961 '}\n') 1007 '}\n')
962 if rd.getVarFlag('do_menuconfig','task'): 1008 f.write('\ndo_kernel_configme:prepend() {\n'
963 f.write('\ndo_configure_append() {\n' 1009 ' if [ -e ${S}/.config ]; then\n'
964 ' if [ ! ${DEVTOOL_DISABLE_MENUCONFIG} ]; then\n' 1010 ' mv ${S}/.config ${S}/.config.old\n'
965 ' cp ${B}/.config ${S}/.config.baseline\n' 1011 ' fi\n'
966 ' ln -sfT ${B}/.config ${S}/.config.new\n' 1012 '}\n')
1013 if rd.getVarFlag('do_menuconfig', 'task'):
1014 f.write('\ndo_configure:append() {\n'
1015 ' if [ ${@oe.types.boolean(d.getVar("KCONFIG_CONFIG_ENABLE_MENUCONFIG"))} = True ]; then\n'
1016 ' cp ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.baseline\n'
1017 ' ln -sfT ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.new\n'
967 ' fi\n' 1018 ' fi\n'
968 '}\n') 1019 '}\n')
969 if initial_rev: 1020 if initial_revs:
970 f.write('\n# initial_rev: %s\n' % initial_rev) 1021 for name, rev in initial_revs.items():
971 for commit in commits: 1022 f.write('\n# initial_rev %s: %s\n' % (name, rev))
972 f.write('# commit: %s\n' % commit) 1023 if name in commits:
1024 for commit in commits[name]:
1025 f.write('# commit %s: %s\n' % (name, commit))
973 if branch_patches: 1026 if branch_patches:
974 for branch in branch_patches: 1027 for branch in branch_patches:
975 if branch == args.branch: 1028 if branch == args.branch:
@@ -1089,10 +1142,10 @@ def rename(args, config, basepath, workspace):
1089 1142
1090 # Rename bbappend 1143 # Rename bbappend
1091 logger.info('Renaming %s to %s' % (append, newappend)) 1144 logger.info('Renaming %s to %s' % (append, newappend))
1092 os.rename(append, newappend) 1145 bb.utils.rename(append, newappend)
1093 # Rename recipe file 1146 # Rename recipe file
1094 logger.info('Renaming %s to %s' % (recipefile, newfile)) 1147 logger.info('Renaming %s to %s' % (recipefile, newfile))
1095 os.rename(recipefile, newfile) 1148 bb.utils.rename(recipefile, newfile)
1096 1149
1097 # Rename source tree if it's the default path 1150 # Rename source tree if it's the default path
1098 appendmd5 = None 1151 appendmd5 = None
@@ -1192,44 +1245,56 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refre
1192 branchname = stdout.rstrip() 1245 branchname = stdout.rstrip()
1193 1246
1194 # Parse initial rev from recipe if not specified 1247 # Parse initial rev from recipe if not specified
1195 commits = [] 1248 commits = {}
1196 patches = [] 1249 patches = []
1250 initial_revs = {}
1197 with open(recipe_path, 'r') as f: 1251 with open(recipe_path, 'r') as f:
1198 for line in f: 1252 for line in f:
1199 if line.startswith('# initial_rev:'): 1253 pattern = r'^#\s.*\s(.*):\s([0-9a-fA-F]+)$'
1200 if not initial_rev: 1254 match = re.search(pattern, line)
1201 initial_rev = line.split(':')[-1].strip() 1255 if match:
1202 elif line.startswith('# commit:') and not force_patch_refresh: 1256 name = match.group(1)
1203 commits.append(line.split(':')[-1].strip()) 1257 rev = match.group(2)
1204 elif line.startswith('# patches_%s:' % branchname): 1258 if line.startswith('# initial_rev'):
1205 patches = line.split(':')[-1].strip().split(',') 1259 if not (name == "." and initial_rev):
1206 1260 initial_revs[name] = rev
1207 update_rev = initial_rev 1261 elif line.startswith('# commit') and not force_patch_refresh:
1208 changed_revs = None 1262 if name not in commits:
1209 if initial_rev: 1263 commits[name] = [rev]
1264 else:
1265 commits[name].append(rev)
1266 elif line.startswith('# patches_%s:' % branchname):
1267 patches = line.split(':')[-1].strip().split(',')
1268
1269 update_revs = dict(initial_revs)
1270 changed_revs = {}
1271 for name, rev in initial_revs.items():
1210 # Find first actually changed revision 1272 # Find first actually changed revision
1211 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' % 1273 stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' %
1212 initial_rev, cwd=srctree) 1274 rev, cwd=os.path.join(srctree, name))
1213 newcommits = stdout.split() 1275 newcommits = stdout.split()
1214 for i in range(min(len(commits), len(newcommits))): 1276 if name in commits:
1215 if newcommits[i] == commits[i]: 1277 for i in range(min(len(commits[name]), len(newcommits))):
1216 update_rev = commits[i] 1278 if newcommits[i] == commits[name][i]:
1279 update_revs[name] = commits[name][i]
1217 1280
1218 try: 1281 try:
1219 stdout, _ = bb.process.run('git cherry devtool-patched', 1282 stdout, _ = bb.process.run('git cherry devtool-patched',
1220 cwd=srctree) 1283 cwd=os.path.join(srctree, name))
1221 except bb.process.ExecutionError as err: 1284 except bb.process.ExecutionError as err:
1222 stdout = None 1285 stdout = None
1223 1286
1224 if stdout is not None and not force_patch_refresh: 1287 if stdout is not None and not force_patch_refresh:
1225 changed_revs = []
1226 for line in stdout.splitlines(): 1288 for line in stdout.splitlines():
1227 if line.startswith('+ '): 1289 if line.startswith('+ '):
1228 rev = line.split()[1] 1290 rev = line.split()[1]
1229 if rev in newcommits: 1291 if rev in newcommits:
1230 changed_revs.append(rev) 1292 if name not in changed_revs:
1293 changed_revs[name] = [rev]
1294 else:
1295 changed_revs[name].append(rev)
1231 1296
1232 return initial_rev, update_rev, changed_revs, patches 1297 return initial_revs, update_revs, changed_revs, patches
1233 1298
1234def _remove_file_entries(srcuri, filelist): 1299def _remove_file_entries(srcuri, filelist):
1235 """Remove file:// entries from SRC_URI""" 1300 """Remove file:// entries from SRC_URI"""
@@ -1284,14 +1349,17 @@ def _remove_source_files(append, files, destpath, no_report_remove=False, dry_ru
1284 raise 1349 raise
1285 1350
1286 1351
1287def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None): 1352def _export_patches(srctree, rd, start_revs, destdir, changed_revs=None):
1288 """Export patches from srctree to given location. 1353 """Export patches from srctree to given location.
1289 Returns three-tuple of dicts: 1354 Returns three-tuple of dicts:
1290 1. updated - patches that already exist in SRCURI 1355 1. updated - patches that already exist in SRCURI
1291 2. added - new patches that don't exist in SRCURI 1356 2. added - new patches that don't exist in SRCURI
1292 3 removed - patches that exist in SRCURI but not in exported patches 1357 3 removed - patches that exist in SRCURI but not in exported patches
1293 In each dict the key is the 'basepath' of the URI and value is the 1358 In each dict the key is the 'basepath' of the URI and value is:
1294 absolute path to the existing file in recipe space (if any). 1359 - for updated and added dicts, a dict with 2 optionnal keys:
1360 - 'path': the absolute path to the existing file in recipe space (if any)
1361 - 'patchdir': the directory in wich the patch should be applied (if any)
1362 - for removed dict, the absolute path to the existing file in recipe space
1295 """ 1363 """
1296 import oe.recipeutils 1364 import oe.recipeutils
1297 from oe.patch import GitApplyTree 1365 from oe.patch import GitApplyTree
@@ -1305,54 +1373,60 @@ def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
1305 1373
1306 # Generate patches from Git, exclude local files directory 1374 # Generate patches from Git, exclude local files directory
1307 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files') 1375 patch_pathspec = _git_exclude_path(srctree, 'oe-local-files')
1308 GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec) 1376 GitApplyTree.extractPatches(srctree, start_revs, destdir, patch_pathspec)
1309 1377 for dirpath, dirnames, filenames in os.walk(destdir):
1310 new_patches = sorted(os.listdir(destdir)) 1378 new_patches = filenames
1311 for new_patch in new_patches: 1379 reldirpath = os.path.relpath(dirpath, destdir)
1312 # Strip numbering from patch names. If it's a git sequence named patch, 1380 for new_patch in new_patches:
1313 # the numbers might not match up since we are starting from a different 1381 # Strip numbering from patch names. If it's a git sequence named patch,
1314 # revision This does assume that people are using unique shortlog 1382 # the numbers might not match up since we are starting from a different
1315 # values, but they ought to be anyway... 1383 # revision This does assume that people are using unique shortlog
1316 new_basename = seqpatch_re.match(new_patch).group(2) 1384 # values, but they ought to be anyway...
1317 match_name = None 1385 new_basename = seqpatch_re.match(new_patch).group(2)
1318 for old_patch in existing_patches: 1386 match_name = None
1319 old_basename = seqpatch_re.match(old_patch).group(2) 1387 for old_patch in existing_patches:
1320 old_basename_splitext = os.path.splitext(old_basename) 1388 old_basename = seqpatch_re.match(old_patch).group(2)
1321 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename: 1389 old_basename_splitext = os.path.splitext(old_basename)
1322 old_patch_noext = os.path.splitext(old_patch)[0] 1390 if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename:
1323 match_name = old_patch_noext 1391 old_patch_noext = os.path.splitext(old_patch)[0]
1324 break 1392 match_name = old_patch_noext
1325 elif new_basename == old_basename: 1393 break
1326 match_name = old_patch 1394 elif new_basename == old_basename:
1327 break 1395 match_name = old_patch
1328 if match_name: 1396 break
1329 # Rename patch files 1397 if match_name:
1330 if new_patch != match_name: 1398 # Rename patch files
1331 os.rename(os.path.join(destdir, new_patch), 1399 if new_patch != match_name:
1332 os.path.join(destdir, match_name)) 1400 bb.utils.rename(os.path.join(destdir, new_patch),
1333 # Need to pop it off the list now before checking changed_revs 1401 os.path.join(destdir, match_name))
1334 oldpath = existing_patches.pop(old_patch) 1402 # Need to pop it off the list now before checking changed_revs
1335 if changed_revs is not None: 1403 oldpath = existing_patches.pop(old_patch)
1336 # Avoid updating patches that have not actually changed 1404 if changed_revs is not None and dirpath in changed_revs:
1337 with open(os.path.join(destdir, match_name), 'r') as f: 1405 # Avoid updating patches that have not actually changed
1338 firstlineitems = f.readline().split() 1406 with open(os.path.join(dirpath, match_name), 'r') as f:
1339 # Looking for "From <hash>" line 1407 firstlineitems = f.readline().split()
1340 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40: 1408 # Looking for "From <hash>" line
1341 if not firstlineitems[1] in changed_revs: 1409 if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40:
1342 continue 1410 if not firstlineitems[1] in changed_revs[dirpath]:
1343 # Recompress if necessary 1411 continue
1344 if oldpath.endswith(('.gz', '.Z')): 1412 # Recompress if necessary
1345 bb.process.run(['gzip', match_name], cwd=destdir) 1413 if oldpath.endswith(('.gz', '.Z')):
1346 if oldpath.endswith('.gz'): 1414 bb.process.run(['gzip', match_name], cwd=destdir)
1347 match_name += '.gz' 1415 if oldpath.endswith('.gz'):
1348 else: 1416 match_name += '.gz'
1349 match_name += '.Z' 1417 else:
1350 elif oldpath.endswith('.bz2'): 1418 match_name += '.Z'
1351 bb.process.run(['bzip2', match_name], cwd=destdir) 1419 elif oldpath.endswith('.bz2'):
1352 match_name += '.bz2' 1420 bb.process.run(['bzip2', match_name], cwd=destdir)
1353 updated[match_name] = oldpath 1421 match_name += '.bz2'
1354 else: 1422 updated[match_name] = {'path' : oldpath}
1355 added[new_patch] = None 1423 if reldirpath != ".":
1424 updated[match_name]['patchdir'] = reldirpath
1425 else:
1426 added[new_patch] = {}
1427 if reldirpath != ".":
1428 added[new_patch]['patchdir'] = reldirpath
1429
1356 return (updated, added, existing_patches) 1430 return (updated, added, existing_patches)
1357 1431
1358 1432
@@ -1389,8 +1463,10 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
1389 1. updated - files that already exist in SRCURI 1463 1. updated - files that already exist in SRCURI
1390 2. added - new files files that don't exist in SRCURI 1464 2. added - new files files that don't exist in SRCURI
1391 3 removed - files that exist in SRCURI but not in exported files 1465 3 removed - files that exist in SRCURI but not in exported files
1392 In each dict the key is the 'basepath' of the URI and value is the 1466 In each dict the key is the 'basepath' of the URI and value is:
1393 absolute path to the existing file in recipe space (if any). 1467 - for updated and added dicts, a dict with 1 optionnal key:
1468 - 'path': the absolute path to the existing file in recipe space (if any)
1469 - for removed dict, the absolute path to the existing file in recipe space
1394 """ 1470 """
1395 import oe.recipeutils 1471 import oe.recipeutils
1396 1472
@@ -1403,6 +1479,18 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
1403 updated = OrderedDict() 1479 updated = OrderedDict()
1404 added = OrderedDict() 1480 added = OrderedDict()
1405 removed = OrderedDict() 1481 removed = OrderedDict()
1482
1483 # Get current branch and return early with empty lists
1484 # if on one of the override branches
1485 # (local files are provided only for the main branch and processing
1486 # them against lists from recipe overrides will result in mismatches
1487 # and broken modifications to recipes).
1488 stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD',
1489 cwd=srctree)
1490 branchname = stdout.rstrip()
1491 if branchname.startswith(override_branch_prefix):
1492 return (updated, added, removed)
1493
1406 local_files_dir = os.path.join(srctreebase, 'oe-local-files') 1494 local_files_dir = os.path.join(srctreebase, 'oe-local-files')
1407 git_files = _git_ls_tree(srctree) 1495 git_files = _git_ls_tree(srctree)
1408 if 'oe-local-files' in git_files: 1496 if 'oe-local-files' in git_files:
@@ -1460,9 +1548,9 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
1460 origpath = existing_files.pop(fname) 1548 origpath = existing_files.pop(fname)
1461 workpath = os.path.join(local_files_dir, fname) 1549 workpath = os.path.join(local_files_dir, fname)
1462 if not filecmp.cmp(origpath, workpath): 1550 if not filecmp.cmp(origpath, workpath):
1463 updated[fname] = origpath 1551 updated[fname] = {'path' : origpath}
1464 elif fname != '.gitignore': 1552 elif fname != '.gitignore':
1465 added[fname] = None 1553 added[fname] = {}
1466 1554
1467 workdir = rd.getVar('WORKDIR') 1555 workdir = rd.getVar('WORKDIR')
1468 s = rd.getVar('S') 1556 s = rd.getVar('S')
@@ -1479,7 +1567,7 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
1479 if os.path.exists(fpath): 1567 if os.path.exists(fpath):
1480 origpath = existing_files.pop(fname) 1568 origpath = existing_files.pop(fname)
1481 if not filecmp.cmp(origpath, fpath): 1569 if not filecmp.cmp(origpath, fpath):
1482 updated[fpath] = origpath 1570 updated[fpath] = {'path' : origpath}
1483 1571
1484 removed = existing_files 1572 removed = existing_files
1485 return (updated, added, removed) 1573 return (updated, added, removed)
@@ -1508,6 +1596,12 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
1508 recipedir = os.path.basename(recipefile) 1596 recipedir = os.path.basename(recipefile)
1509 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix)) 1597 logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix))
1510 1598
1599 # Get original SRCREV
1600 old_srcrev = rd.getVar('SRCREV') or ''
1601 if old_srcrev == "INVALID":
1602 raise DevtoolError('Update mode srcrev is only valid for recipe fetched from an SCM repository')
1603 old_srcrev = {'.': old_srcrev}
1604
1511 # Get HEAD revision 1605 # Get HEAD revision
1512 try: 1606 try:
1513 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree) 1607 stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree)
@@ -1534,13 +1628,12 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
1534 if not no_remove: 1628 if not no_remove:
1535 # Find list of existing patches in recipe file 1629 # Find list of existing patches in recipe file
1536 patches_dir = tempfile.mkdtemp(dir=tempdir) 1630 patches_dir = tempfile.mkdtemp(dir=tempdir)
1537 old_srcrev = rd.getVar('SRCREV') or ''
1538 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev, 1631 upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev,
1539 patches_dir) 1632 patches_dir)
1540 logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p))) 1633 logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p)))
1541 1634
1542 # Remove deleted local files and "overlapping" patches 1635 # Remove deleted local files and "overlapping" patches
1543 remove_files = list(del_f.values()) + list(upd_p.values()) + list(del_p.values()) 1636 remove_files = list(del_f.values()) + [value["path"] for value in upd_p.values() if "path" in value] + [value["path"] for value in del_p.values() if "path" in value]
1544 if remove_files: 1637 if remove_files:
1545 removedentries = _remove_file_entries(srcuri, remove_files)[0] 1638 removedentries = _remove_file_entries(srcuri, remove_files)[0]
1546 update_srcuri = True 1639 update_srcuri = True
@@ -1554,14 +1647,14 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
1554 patchfields['SRC_URI'] = '\\\n '.join(srcuri) 1647 patchfields['SRC_URI'] = '\\\n '.join(srcuri)
1555 if dry_run_outdir: 1648 if dry_run_outdir:
1556 logger.info('Creating bbappend (dry-run)') 1649 logger.info('Creating bbappend (dry-run)')
1557 else: 1650 appendfile, destpath = oe.recipeutils.bbappend_recipe(
1558 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1651 rd, appendlayerdir, files, wildcardver=wildcard_version,
1559 rd, appendlayerdir, files, wildcardver=wildcard_version, 1652 extralines=patchfields, removevalues=removevalues,
1560 extralines=patchfields, removevalues=removevalues, 1653 redirect_output=dry_run_outdir)
1561 redirect_output=dry_run_outdir)
1562 else: 1654 else:
1563 files_dir = _determine_files_dir(rd) 1655 files_dir = _determine_files_dir(rd)
1564 for basepath, path in upd_f.items(): 1656 for basepath, param in upd_f.items():
1657 path = param['path']
1565 logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) 1658 logger.info('Updating file %s%s' % (basepath, dry_run_suffix))
1566 if os.path.isabs(basepath): 1659 if os.path.isabs(basepath):
1567 # Original file (probably with subdir pointing inside source tree) 1660 # Original file (probably with subdir pointing inside source tree)
@@ -1571,7 +1664,8 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
1571 _move_file(os.path.join(local_files_dir, basepath), path, 1664 _move_file(os.path.join(local_files_dir, basepath), path,
1572 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1665 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1573 update_srcuri= True 1666 update_srcuri= True
1574 for basepath, path in new_f.items(): 1667 for basepath, param in new_f.items():
1668 path = param['path']
1575 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1669 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
1576 _move_file(os.path.join(local_files_dir, basepath), 1670 _move_file(os.path.join(local_files_dir, basepath),
1577 os.path.join(files_dir, basepath), 1671 os.path.join(files_dir, basepath),
@@ -1603,9 +1697,22 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
1603 if not os.path.exists(append): 1697 if not os.path.exists(append):
1604 raise DevtoolError('unable to find workspace bbappend for recipe %s' % 1698 raise DevtoolError('unable to find workspace bbappend for recipe %s' %
1605 recipename) 1699 recipename)
1700 srctreebase = workspace[recipename]['srctreebase']
1701 relpatchdir = os.path.relpath(srctreebase, srctree)
1702 if relpatchdir == '.':
1703 patchdir_params = {}
1704 else:
1705 patchdir_params = {'patchdir': relpatchdir}
1606 1706
1607 initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh) 1707 def srcuri_entry(basepath, patchdir_params):
1608 if not initial_rev: 1708 if patchdir_params:
1709 paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items())
1710 else:
1711 paramstr = ''
1712 return 'file://%s%s' % (basepath, paramstr)
1713
1714 initial_revs, update_revs, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh)
1715 if not initial_revs:
1609 raise DevtoolError('Unable to find initial revision - please specify ' 1716 raise DevtoolError('Unable to find initial revision - please specify '
1610 'it with --initial-rev') 1717 'it with --initial-rev')
1611 1718
@@ -1619,61 +1726,69 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
1619 tempdir = tempfile.mkdtemp(prefix='devtool') 1726 tempdir = tempfile.mkdtemp(prefix='devtool')
1620 try: 1727 try:
1621 local_files_dir = tempfile.mkdtemp(dir=tempdir) 1728 local_files_dir = tempfile.mkdtemp(dir=tempdir)
1622 if filter_patches: 1729 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
1623 upd_f = {}
1624 new_f = {}
1625 del_f = {}
1626 else:
1627 srctreebase = workspace[recipename]['srctreebase']
1628 upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
1629
1630 remove_files = []
1631 if not no_remove:
1632 # Get all patches from source tree and check if any should be removed
1633 all_patches_dir = tempfile.mkdtemp(dir=tempdir)
1634 _, _, del_p = _export_patches(srctree, rd, initial_rev,
1635 all_patches_dir)
1636 # Remove deleted local files and patches
1637 remove_files = list(del_f.values()) + list(del_p.values())
1638 1730
1639 # Get updated patches from source tree 1731 # Get updated patches from source tree
1640 patches_dir = tempfile.mkdtemp(dir=tempdir) 1732 patches_dir = tempfile.mkdtemp(dir=tempdir)
1641 upd_p, new_p, _ = _export_patches(srctree, rd, update_rev, 1733 upd_p, new_p, _ = _export_patches(srctree, rd, update_revs,
1642 patches_dir, changed_revs) 1734 patches_dir, changed_revs)
1735 # Get all patches from source tree and check if any should be removed
1736 all_patches_dir = tempfile.mkdtemp(dir=tempdir)
1737 _, _, del_p = _export_patches(srctree, rd, initial_revs,
1738 all_patches_dir)
1643 logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p))) 1739 logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p)))
1644 if filter_patches: 1740 if filter_patches:
1645 new_p = OrderedDict() 1741 new_p = OrderedDict()
1646 upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches) 1742 upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches)
1647 remove_files = [f for f in remove_files if f in filter_patches] 1743 del_p = OrderedDict((k,v) for k,v in del_p.items() if k in filter_patches)
1744 remove_files = []
1745 if not no_remove:
1746 # Remove deleted local files and patches
1747 remove_files = list(del_f.values()) + list(del_p.values())
1648 updatefiles = False 1748 updatefiles = False
1649 updaterecipe = False 1749 updaterecipe = False
1650 destpath = None 1750 destpath = None
1651 srcuri = (rd.getVar('SRC_URI', False) or '').split() 1751 srcuri = (rd.getVar('SRC_URI', False) or '').split()
1752
1652 if appendlayerdir: 1753 if appendlayerdir:
1653 files = OrderedDict((os.path.join(local_files_dir, key), val) for 1754 files = OrderedDict((os.path.join(local_files_dir, key), val) for
1654 key, val in list(upd_f.items()) + list(new_f.items())) 1755 key, val in list(upd_f.items()) + list(new_f.items()))
1655 files.update(OrderedDict((os.path.join(patches_dir, key), val) for 1756 files.update(OrderedDict((os.path.join(patches_dir, key), val) for
1656 key, val in list(upd_p.items()) + list(new_p.items()))) 1757 key, val in list(upd_p.items()) + list(new_p.items())))
1758
1759 params = []
1760 for file, param in files.items():
1761 patchdir_param = dict(patchdir_params)
1762 patchdir = param.get('patchdir', ".")
1763 if patchdir != "." :
1764 if patchdir_param:
1765 patchdir_param['patchdir'] += patchdir
1766 else:
1767 patchdir_param['patchdir'] = patchdir
1768 params.append(patchdir_param)
1769
1657 if files or remove_files: 1770 if files or remove_files:
1658 removevalues = None 1771 removevalues = None
1659 if remove_files: 1772 if remove_files:
1660 removedentries, remaining = _remove_file_entries( 1773 removedentries, remaining = _remove_file_entries(
1661 srcuri, remove_files) 1774 srcuri, remove_files)
1662 if removedentries or remaining: 1775 if removedentries or remaining:
1663 remaining = ['file://' + os.path.basename(item) for 1776 remaining = [srcuri_entry(os.path.basename(item), patchdir_params) for
1664 item in remaining] 1777 item in remaining]
1665 removevalues = {'SRC_URI': removedentries + remaining} 1778 removevalues = {'SRC_URI': removedentries + remaining}
1666 appendfile, destpath = oe.recipeutils.bbappend_recipe( 1779 appendfile, destpath = oe.recipeutils.bbappend_recipe(
1667 rd, appendlayerdir, files, 1780 rd, appendlayerdir, files,
1668 wildcardver=wildcard_version, 1781 wildcardver=wildcard_version,
1669 removevalues=removevalues, 1782 removevalues=removevalues,
1670 redirect_output=dry_run_outdir) 1783 redirect_output=dry_run_outdir,
1784 params=params)
1671 else: 1785 else:
1672 logger.info('No patches or local source files needed updating') 1786 logger.info('No patches or local source files needed updating')
1673 else: 1787 else:
1674 # Update existing files 1788 # Update existing files
1675 files_dir = _determine_files_dir(rd) 1789 files_dir = _determine_files_dir(rd)
1676 for basepath, path in upd_f.items(): 1790 for basepath, param in upd_f.items():
1791 path = param['path']
1677 logger.info('Updating file %s' % basepath) 1792 logger.info('Updating file %s' % basepath)
1678 if os.path.isabs(basepath): 1793 if os.path.isabs(basepath):
1679 # Original file (probably with subdir pointing inside source tree) 1794 # Original file (probably with subdir pointing inside source tree)
@@ -1684,14 +1799,22 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
1684 _move_file(os.path.join(local_files_dir, basepath), path, 1799 _move_file(os.path.join(local_files_dir, basepath), path,
1685 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1800 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1686 updatefiles = True 1801 updatefiles = True
1687 for basepath, path in upd_p.items(): 1802 for basepath, param in upd_p.items():
1688 patchfn = os.path.join(patches_dir, basepath) 1803 path = param['path']
1804 patchdir = param.get('patchdir', ".")
1805 if patchdir != "." :
1806 patchdir_param = dict(patchdir_params)
1807 if patchdir_param:
1808 patchdir_param['patchdir'] += patchdir
1809 else:
1810 patchdir_param['patchdir'] = patchdir
1811 patchfn = os.path.join(patches_dir, patchdir, basepath)
1689 if os.path.dirname(path) + '/' == dl_dir: 1812 if os.path.dirname(path) + '/' == dl_dir:
1690 # This is a a downloaded patch file - we now need to 1813 # This is a a downloaded patch file - we now need to
1691 # replace the entry in SRC_URI with our local version 1814 # replace the entry in SRC_URI with our local version
1692 logger.info('Replacing remote patch %s with updated local version' % basepath) 1815 logger.info('Replacing remote patch %s with updated local version' % basepath)
1693 path = os.path.join(files_dir, basepath) 1816 path = os.path.join(files_dir, basepath)
1694 _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath) 1817 _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath, patchdir_param))
1695 updaterecipe = True 1818 updaterecipe = True
1696 else: 1819 else:
1697 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix)) 1820 logger.info('Updating patch %s%s' % (basepath, dry_run_suffix))
@@ -1699,21 +1822,29 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
1699 dry_run_outdir=dry_run_outdir, base_outdir=recipedir) 1822 dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
1700 updatefiles = True 1823 updatefiles = True
1701 # Add any new files 1824 # Add any new files
1702 for basepath, path in new_f.items(): 1825 for basepath, param in new_f.items():
1703 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) 1826 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
1704 _move_file(os.path.join(local_files_dir, basepath), 1827 _move_file(os.path.join(local_files_dir, basepath),
1705 os.path.join(files_dir, basepath), 1828 os.path.join(files_dir, basepath),
1706 dry_run_outdir=dry_run_outdir, 1829 dry_run_outdir=dry_run_outdir,
1707 base_outdir=recipedir) 1830 base_outdir=recipedir)
1708 srcuri.append('file://%s' % basepath) 1831 srcuri.append(srcuri_entry(basepath, patchdir_params))
1709 updaterecipe = True 1832 updaterecipe = True
1710 for basepath, path in new_p.items(): 1833 for basepath, param in new_p.items():
1834 patchdir = param.get('patchdir', ".")
1711 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix)) 1835 logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix))
1712 _move_file(os.path.join(patches_dir, basepath), 1836 _move_file(os.path.join(patches_dir, patchdir, basepath),
1713 os.path.join(files_dir, basepath), 1837 os.path.join(files_dir, basepath),
1714 dry_run_outdir=dry_run_outdir, 1838 dry_run_outdir=dry_run_outdir,
1715 base_outdir=recipedir) 1839 base_outdir=recipedir)
1716 srcuri.append('file://%s' % basepath) 1840 params = dict(patchdir_params)
1841 if patchdir != "." :
1842 if params:
1843 params['patchdir'] += patchdir
1844 else:
1845 params['patchdir'] = patchdir
1846
1847 srcuri.append(srcuri_entry(basepath, params))
1717 updaterecipe = True 1848 updaterecipe = True
1718 # Update recipe, if needed 1849 # Update recipe, if needed
1719 if _remove_file_entries(srcuri, remove_files)[0]: 1850 if _remove_file_entries(srcuri, remove_files)[0]:
@@ -1770,6 +1901,8 @@ def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_ver
1770 for line in stdout.splitlines(): 1901 for line in stdout.splitlines():
1771 branchname = line[2:] 1902 branchname = line[2:]
1772 if line.startswith('* '): 1903 if line.startswith('* '):
1904 if 'HEAD' in line:
1905 raise DevtoolError('Detached HEAD - please check out a branch, e.g., "devtool"')
1773 startbranch = branchname 1906 startbranch = branchname
1774 if branchname.startswith(override_branch_prefix): 1907 if branchname.startswith(override_branch_prefix):
1775 override_branches.append(branchname) 1908 override_branches.append(branchname)
@@ -1959,9 +2092,19 @@ def _reset(recipes, no_clean, remove_work, config, basepath, workspace):
1959 shutil.rmtree(srctreebase) 2092 shutil.rmtree(srctreebase)
1960 else: 2093 else:
1961 # We don't want to risk wiping out any work in progress 2094 # We don't want to risk wiping out any work in progress
1962 logger.info('Leaving source tree %s as-is; if you no ' 2095 if srctreebase.startswith(os.path.join(config.workspace_path, 'sources')):
1963 'longer need it then please delete it manually' 2096 from datetime import datetime
1964 % srctreebase) 2097 preservesrc = os.path.join(config.workspace_path, 'attic', 'sources', "{}.{}".format(pn, datetime.now().strftime("%Y%m%d%H%M%S")))
2098 logger.info('Preserving source tree in %s\nIf you no '
2099 'longer need it then please delete it manually.\n'
2100 'It is also possible to reuse it via devtool source tree argument.'
2101 % preservesrc)
2102 bb.utils.mkdirhier(os.path.dirname(preservesrc))
2103 shutil.move(srctreebase, preservesrc)
2104 else:
2105 logger.info('Leaving source tree %s as-is; if you no '
2106 'longer need it then please delete it manually'
2107 % srctreebase)
1965 else: 2108 else:
1966 # This is unlikely, but if it's empty we can just remove it 2109 # This is unlikely, but if it's empty we can just remove it
1967 os.rmdir(srctreebase) 2110 os.rmdir(srctreebase)
@@ -2221,6 +2364,7 @@ def register_commands(subparsers, context):
2221 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true") 2364 group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
2222 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI') 2365 parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI')
2223 parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true") 2366 parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true")
2367 parser_add.add_argument('--no-pypi', help='Do not inherit pypi class', action="store_true")
2224 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)') 2368 parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
2225 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true") 2369 parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true")
2226 group = parser_add.add_mutually_exclusive_group() 2370 group = parser_add.add_mutually_exclusive_group()
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 5a057e95f5..fa5b8ef3c7 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -35,6 +35,8 @@ def _get_srctree(tmpdir):
35 dirs = scriptutils.filter_src_subdirs(tmpdir) 35 dirs = scriptutils.filter_src_subdirs(tmpdir)
36 if len(dirs) == 1: 36 if len(dirs) == 1:
37 srctree = os.path.join(tmpdir, dirs[0]) 37 srctree = os.path.join(tmpdir, dirs[0])
38 else:
39 raise DevtoolError("Cannot determine where the source tree is after unpacking in {}: {}".format(tmpdir,dirs))
38 return srctree 40 return srctree
39 41
40def _copy_source_code(orig, dest): 42def _copy_source_code(orig, dest):
@@ -71,7 +73,8 @@ def _rename_recipe_dirs(oldpv, newpv, path):
71 if oldfile.find(oldpv) != -1: 73 if oldfile.find(oldpv) != -1:
72 newfile = oldfile.replace(oldpv, newpv) 74 newfile = oldfile.replace(oldpv, newpv)
73 if oldfile != newfile: 75 if oldfile != newfile:
74 os.rename(os.path.join(path, oldfile), os.path.join(path, newfile)) 76 bb.utils.rename(os.path.join(path, oldfile),
77 os.path.join(path, newfile))
75 78
76def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path): 79def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path):
77 oldrecipe = os.path.basename(oldrecipe) 80 oldrecipe = os.path.basename(oldrecipe)
@@ -87,7 +90,7 @@ def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path):
87 _rename_recipe_dirs(oldpv, newpv, path) 90 _rename_recipe_dirs(oldpv, newpv, path)
88 return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path) 91 return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path)
89 92
90def _write_append(rc, srctree, same_dir, no_same_dir, rev, copied, workspace, d): 93def _write_append(rc, srctreebase, srctree, same_dir, no_same_dir, revs, copied, workspace, d):
91 """Writes an append file""" 94 """Writes an append file"""
92 if not os.path.exists(rc): 95 if not os.path.exists(rc):
93 raise DevtoolError("bbappend not created because %s does not exist" % rc) 96 raise DevtoolError("bbappend not created because %s does not exist" % rc)
@@ -102,36 +105,38 @@ def _write_append(rc, srctree, same_dir, no_same_dir, rev, copied, workspace, d)
102 pn = d.getVar('PN') 105 pn = d.getVar('PN')
103 af = os.path.join(appendpath, '%s.bbappend' % brf) 106 af = os.path.join(appendpath, '%s.bbappend' % brf)
104 with open(af, 'w') as f: 107 with open(af, 'w') as f:
105 f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n') 108 f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n\n')
109 # Local files can be modified/tracked in separate subdir under srctree
110 # Mostly useful for packages with S != WORKDIR
111 f.write('FILESPATH:prepend := "%s:"\n' %
112 os.path.join(srctreebase, 'oe-local-files'))
113 f.write('# srctreebase: %s\n' % srctreebase)
106 f.write('inherit externalsrc\n') 114 f.write('inherit externalsrc\n')
107 f.write(('# NOTE: We use pn- overrides here to avoid affecting' 115 f.write(('# NOTE: We use pn- overrides here to avoid affecting'
108 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n')) 116 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n'))
109 f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree)) 117 f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree))
110 b_is_s = use_external_build(same_dir, no_same_dir, d) 118 b_is_s = use_external_build(same_dir, no_same_dir, d)
111 if b_is_s: 119 if b_is_s:
112 f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree)) 120 f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree))
113 f.write('\n') 121 f.write('\n')
114 if rev: 122 if revs:
115 f.write('# initial_rev: %s\n' % rev) 123 for name, rev in revs.items():
124 f.write('# initial_rev %s: %s\n' % (name, rev))
116 if copied: 125 if copied:
117 f.write('# original_path: %s\n' % os.path.dirname(d.getVar('FILE'))) 126 f.write('# original_path: %s\n' % os.path.dirname(d.getVar('FILE')))
118 f.write('# original_files: %s\n' % ' '.join(copied)) 127 f.write('# original_files: %s\n' % ' '.join(copied))
119 return af 128 return af
120 129
121def _cleanup_on_error(rf, srctree): 130def _cleanup_on_error(rd, srctree):
122 rfp = os.path.split(rf)[0] # recipe folder 131 if os.path.exists(rd):
123 rfpp = os.path.split(rfp)[0] # recipes folder 132 shutil.rmtree(rd)
124 if os.path.exists(rfp):
125 shutil.rmtree(rfp)
126 if not len(os.listdir(rfpp)):
127 os.rmdir(rfpp)
128 srctree = os.path.abspath(srctree) 133 srctree = os.path.abspath(srctree)
129 if os.path.exists(srctree): 134 if os.path.exists(srctree):
130 shutil.rmtree(srctree) 135 shutil.rmtree(srctree)
131 136
132def _upgrade_error(e, rf, srctree, keep_failure=False, extramsg=None): 137def _upgrade_error(e, rd, srctree, keep_failure=False, extramsg=None):
133 if rf and not keep_failure: 138 if not keep_failure:
134 _cleanup_on_error(rf, srctree) 139 _cleanup_on_error(rd, srctree)
135 logger.error(e) 140 logger.error(e)
136 if extramsg: 141 if extramsg:
137 logger.error(extramsg) 142 logger.error(extramsg)
@@ -178,12 +183,16 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
178 uri, rev = _get_uri(crd) 183 uri, rev = _get_uri(crd)
179 if srcrev: 184 if srcrev:
180 rev = srcrev 185 rev = srcrev
186 paths = [srctree]
181 if uri.startswith('git://') or uri.startswith('gitsm://'): 187 if uri.startswith('git://') or uri.startswith('gitsm://'):
182 __run('git fetch') 188 __run('git fetch')
183 __run('git checkout %s' % rev) 189 __run('git checkout %s' % rev)
184 __run('git tag -f devtool-base-new') 190 __run('git tag -f devtool-base-new')
185 md5 = None 191 __run('git submodule update --recursive')
186 sha256 = None 192 __run('git submodule foreach \'git tag -f devtool-base-new\'')
193 (stdout, _) = __run('git submodule --quiet foreach \'echo $sm_path\'')
194 paths += [os.path.join(srctree, p) for p in stdout.splitlines()]
195 checksums = {}
187 _, _, _, _, _, params = bb.fetch2.decodeurl(uri) 196 _, _, _, _, _, params = bb.fetch2.decodeurl(uri)
188 srcsubdir_rel = params.get('destsuffix', 'git') 197 srcsubdir_rel = params.get('destsuffix', 'git')
189 if not srcbranch: 198 if not srcbranch:
@@ -191,14 +200,15 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
191 get_branch = [x.strip() for x in check_branch.splitlines()] 200 get_branch = [x.strip() for x in check_branch.splitlines()]
192 # Remove HEAD reference point and drop remote prefix 201 # Remove HEAD reference point and drop remote prefix
193 get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')] 202 get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')]
194 if 'master' in get_branch: 203 if len(get_branch) == 1:
195 # If it is master, we do not need to append 'branch=master' as this is default. 204 # If srcrev is on only ONE branch, then use that branch
196 # Even with the case where get_branch has multiple objects, if 'master' is one
197 # of them, we should default take from 'master'
198 srcbranch = ''
199 elif len(get_branch) == 1:
200 # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch'
201 srcbranch = get_branch[0] 205 srcbranch = get_branch[0]
206 elif 'main' in get_branch:
207 # If srcrev is on multiple branches, then choose 'main' if it is one of them
208 srcbranch = 'main'
209 elif 'master' in get_branch:
210 # Otherwise choose 'master' if it is one of the branches
211 srcbranch = 'master'
202 else: 212 else:
203 # If get_branch contains more than one objects, then display error and exit. 213 # If get_branch contains more than one objects, then display error and exit.
204 mbrch = '\n ' + '\n '.join(get_branch) 214 mbrch = '\n ' + '\n '.join(get_branch)
@@ -215,9 +225,6 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
215 if ftmpdir and keep_temp: 225 if ftmpdir and keep_temp:
216 logger.info('Fetch temp directory is %s' % ftmpdir) 226 logger.info('Fetch temp directory is %s' % ftmpdir)
217 227
218 md5 = checksums['md5sum']
219 sha256 = checksums['sha256sum']
220
221 tmpsrctree = _get_srctree(tmpdir) 228 tmpsrctree = _get_srctree(tmpdir)
222 srctree = os.path.abspath(srctree) 229 srctree = os.path.abspath(srctree)
223 srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir) 230 srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir)
@@ -251,30 +258,50 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
251 __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv)) 258 __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv))
252 __run('git tag -f devtool-base-%s' % newpv) 259 __run('git tag -f devtool-base-%s' % newpv)
253 260
254 (stdout, _) = __run('git rev-parse HEAD') 261 revs = {}
255 rev = stdout.rstrip() 262 for path in paths:
263 (stdout, _) = _run('git rev-parse HEAD', cwd=path)
264 revs[os.path.relpath(path, srctree)] = stdout.rstrip()
256 265
257 if no_patch: 266 if no_patch:
258 patches = oe.recipeutils.get_recipe_patches(crd) 267 patches = oe.recipeutils.get_recipe_patches(crd)
259 if patches: 268 if patches:
260 logger.warning('By user choice, the following patches will NOT be applied to the new source tree:\n %s' % '\n '.join([os.path.basename(patch) for patch in patches])) 269 logger.warning('By user choice, the following patches will NOT be applied to the new source tree:\n %s' % '\n '.join([os.path.basename(patch) for patch in patches]))
261 else: 270 else:
262 __run('git checkout devtool-patched -b %s' % branch) 271 for path in paths:
263 skiptag = False 272 _run('git checkout devtool-patched -b %s' % branch, cwd=path)
264 try: 273 (stdout, _) = _run('git branch --list devtool-override-*', cwd=path)
265 __run('git rebase %s' % rev) 274 branches_to_rebase = [branch] + stdout.split()
266 except bb.process.ExecutionError as e: 275 target_branch = revs[os.path.relpath(path, srctree)]
267 skiptag = True 276
268 if 'conflict' in e.stdout: 277 # There is a bug (or feature?) in git rebase where if a commit with
269 logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip())) 278 # a note is fully rebased away by being part of an old commit, the
270 else: 279 # note is still attached to the old commit. Avoid this by making
271 logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) 280 # sure all old devtool related commits have a note attached to them
272 if not skiptag: 281 # (this assumes git config notes.rewriteMode is set to ignore).
273 if uri.startswith('git://') or uri.startswith('gitsm://'): 282 (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch)
274 suffix = 'new' 283 for rev in stdout.splitlines():
275 else: 284 if not oe.patch.GitApplyTree.getNotes(path, rev):
276 suffix = newpv 285 oe.patch.GitApplyTree.addNote(path, rev, "dummy")
277 __run('git tag -f devtool-patched-%s' % suffix) 286
287 for b in branches_to_rebase:
288 logger.info("Rebasing {} onto {}".format(b, target_branch))
289 _run('git checkout %s' % b, cwd=path)
290 try:
291 _run('git rebase %s' % target_branch, cwd=path)
292 except bb.process.ExecutionError as e:
293 if 'conflict' in e.stdout:
294 logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip()))
295 _run('git rebase --abort', cwd=path)
296 else:
297 logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
298
299 # Remove any dummy notes added above.
300 (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch)
301 for rev in stdout.splitlines():
302 oe.patch.GitApplyTree.removeNote(path, rev, "dummy")
303
304 _run('git checkout %s' % branch, cwd=path)
278 305
279 if tmpsrctree: 306 if tmpsrctree:
280 if keep_temp: 307 if keep_temp:
@@ -284,7 +311,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
284 if tmpdir != tmpsrctree: 311 if tmpdir != tmpsrctree:
285 shutil.rmtree(tmpdir) 312 shutil.rmtree(tmpdir)
286 313
287 return (rev, md5, sha256, srcbranch, srcsubdir_rel) 314 return (revs, checksums, srcbranch, srcsubdir_rel)
288 315
289def _add_license_diff_to_recipe(path, diff): 316def _add_license_diff_to_recipe(path, diff):
290 notice_text = """# FIXME: the LIC_FILES_CHKSUM values have been updated by 'devtool upgrade'. 317 notice_text = """# FIXME: the LIC_FILES_CHKSUM values have been updated by 'devtool upgrade'.
@@ -305,7 +332,7 @@ def _add_license_diff_to_recipe(path, diff):
305 f.write("\n#\n\n".encode()) 332 f.write("\n#\n\n".encode())
306 f.write(orig_content) 333 f.write(orig_content)
307 334
308def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure): 335def _create_new_recipe(newpv, checksums, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure):
309 """Creates the new recipe under workspace""" 336 """Creates the new recipe under workspace"""
310 337
311 bpn = rd.getVar('BPN') 338 bpn = rd.getVar('BPN')
@@ -336,7 +363,10 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src
336 replacing = True 363 replacing = True
337 new_src_uri = [] 364 new_src_uri = []
338 for entry in src_uri: 365 for entry in src_uri:
339 scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry) 366 try:
367 scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry)
368 except bb.fetch2.MalformedUrl as e:
369 raise DevtoolError("Could not decode SRC_URI: {}".format(e))
340 if replacing and scheme in ['git', 'gitsm']: 370 if replacing and scheme in ['git', 'gitsm']:
341 branch = params.get('branch', 'master') 371 branch = params.get('branch', 'master')
342 if rd.expand(branch) != srcbranch: 372 if rd.expand(branch) != srcbranch:
@@ -374,30 +404,39 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src
374 addnames.append(params['name']) 404 addnames.append(params['name'])
375 # Find what's been set in the original recipe 405 # Find what's been set in the original recipe
376 oldnames = [] 406 oldnames = []
407 oldsums = []
377 noname = False 408 noname = False
378 for varflag in rd.getVarFlags('SRC_URI'): 409 for varflag in rd.getVarFlags('SRC_URI'):
379 if varflag.endswith(('.md5sum', '.sha256sum')): 410 for checksum in checksums:
380 name = varflag.rsplit('.', 1)[0] 411 if varflag.endswith('.' + checksum):
381 if name not in oldnames: 412 name = varflag.rsplit('.', 1)[0]
382 oldnames.append(name) 413 if name not in oldnames:
383 elif varflag in ['md5sum', 'sha256sum']: 414 oldnames.append(name)
384 noname = True 415 oldsums.append(checksum)
416 elif varflag == checksum:
417 noname = True
418 oldsums.append(checksum)
385 # Even if SRC_URI has named entries it doesn't have to actually use the name 419 # Even if SRC_URI has named entries it doesn't have to actually use the name
386 if noname and addnames and addnames[0] not in oldnames: 420 if noname and addnames and addnames[0] not in oldnames:
387 addnames = [] 421 addnames = []
388 # Drop any old names (the name actually might include ${PV}) 422 # Drop any old names (the name actually might include ${PV})
389 for name in oldnames: 423 for name in oldnames:
390 if name not in newnames: 424 if name not in newnames:
391 newvalues['SRC_URI[%s.md5sum]' % name] = None 425 for checksum in oldsums:
392 newvalues['SRC_URI[%s.sha256sum]' % name] = None 426 newvalues['SRC_URI[%s.%s]' % (name, checksum)] = None
393 427
394 if sha256: 428 nameprefix = '%s.' % addnames[0] if addnames else ''
395 if addnames: 429
396 nameprefix = '%s.' % addnames[0] 430 # md5sum is deprecated, remove any traces of it. If it was the only old
397 else: 431 # checksum, then replace it with the default checksums.
398 nameprefix = '' 432 if 'md5sum' in oldsums:
399 newvalues['SRC_URI[%smd5sum]' % nameprefix] = None 433 newvalues['SRC_URI[%smd5sum]' % nameprefix] = None
400 newvalues['SRC_URI[%ssha256sum]' % nameprefix] = sha256 434 oldsums.remove('md5sum')
435 if not oldsums:
436 oldsums = ["%ssum" % s for s in bb.fetch2.SHOWN_CHECKSUM_LIST]
437
438 for checksum in oldsums:
439 newvalues['SRC_URI[%s%s]' % (nameprefix, checksum)] = checksums[checksum]
401 440
402 if srcsubdir_new != srcsubdir_old: 441 if srcsubdir_new != srcsubdir_old:
403 s_subdir_old = os.path.relpath(os.path.abspath(rd.getVar('S')), rd.getVar('WORKDIR')) 442 s_subdir_old = os.path.relpath(os.path.abspath(rd.getVar('S')), rd.getVar('WORKDIR'))
@@ -422,10 +461,11 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src
422 newvalues["LIC_FILES_CHKSUM"] = newlicchksum 461 newvalues["LIC_FILES_CHKSUM"] = newlicchksum
423 _add_license_diff_to_recipe(fullpath, license_diff) 462 _add_license_diff_to_recipe(fullpath, license_diff)
424 463
464 tinfoil.modified_files()
425 try: 465 try:
426 rd = tinfoil.parse_recipe_file(fullpath, False) 466 rd = tinfoil.parse_recipe_file(fullpath, False)
427 except bb.tinfoil.TinfoilCommandFailed as e: 467 except bb.tinfoil.TinfoilCommandFailed as e:
428 _upgrade_error(e, fullpath, srctree, keep_failure, 'Parsing of upgraded recipe failed') 468 _upgrade_error(e, os.path.dirname(fullpath), srctree, keep_failure, 'Parsing of upgraded recipe failed')
429 oe.recipeutils.patch_recipe(rd, fullpath, newvalues) 469 oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
430 470
431 return fullpath, copied 471 return fullpath, copied
@@ -434,7 +474,7 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src
434def _check_git_config(): 474def _check_git_config():
435 def getconfig(name): 475 def getconfig(name):
436 try: 476 try:
437 value = bb.process.run('git config --global %s' % name)[0].strip() 477 value = bb.process.run('git config %s' % name)[0].strip()
438 except bb.process.ExecutionError as e: 478 except bb.process.ExecutionError as e:
439 if e.exitcode == 1: 479 if e.exitcode == 1:
440 value = None 480 value = None
@@ -521,6 +561,8 @@ def upgrade(args, config, basepath, workspace):
521 else: 561 else:
522 srctree = standard.get_default_srctree(config, pn) 562 srctree = standard.get_default_srctree(config, pn)
523 563
564 srctree_s = standard.get_real_srctree(srctree, rd.getVar('S'), rd.getVar('WORKDIR'))
565
524 # try to automatically discover latest version and revision if not provided on command line 566 # try to automatically discover latest version and revision if not provided on command line
525 if not args.version and not args.srcrev: 567 if not args.version and not args.srcrev:
526 version_info = oe.recipeutils.get_recipe_upstream_version(rd) 568 version_info = oe.recipeutils.get_recipe_upstream_version(rd)
@@ -550,21 +592,20 @@ def upgrade(args, config, basepath, workspace):
550 try: 592 try:
551 logger.info('Extracting current version source...') 593 logger.info('Extracting current version source...')
552 rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides) 594 rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
553 old_licenses = _extract_licenses(srctree, (rd.getVar('LIC_FILES_CHKSUM') or "")) 595 old_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or ""))
554 logger.info('Extracting upgraded version source...') 596 logger.info('Extracting upgraded version source...')
555 rev2, md5, sha256, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch, 597 rev2, checksums, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch,
556 args.srcrev, args.srcbranch, args.branch, args.keep_temp, 598 args.srcrev, args.srcbranch, args.branch, args.keep_temp,
557 tinfoil, rd) 599 tinfoil, rd)
558 new_licenses = _extract_licenses(srctree, (rd.getVar('LIC_FILES_CHKSUM') or "")) 600 new_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or ""))
559 license_diff = _generate_license_diff(old_licenses, new_licenses) 601 license_diff = _generate_license_diff(old_licenses, new_licenses)
560 rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure) 602 rf, copied = _create_new_recipe(args.version, checksums, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure)
561 except bb.process.CmdError as e: 603 except (bb.process.CmdError, DevtoolError) as e:
562 _upgrade_error(e, rf, srctree, args.keep_failure) 604 recipedir = os.path.join(config.workspace_path, 'recipes', rd.getVar('BPN'))
563 except DevtoolError as e: 605 _upgrade_error(e, recipedir, srctree, args.keep_failure)
564 _upgrade_error(e, rf, srctree, args.keep_failure)
565 standard._add_md5(config, pn, os.path.dirname(rf)) 606 standard._add_md5(config, pn, os.path.dirname(rf))
566 607
567 af = _write_append(rf, srctree, args.same_dir, args.no_same_dir, rev2, 608 af = _write_append(rf, srctree, srctree_s, args.same_dir, args.no_same_dir, rev2,
568 copied, config.workspace_path, rd) 609 copied, config.workspace_path, rd)
569 standard._add_md5(config, pn, af) 610 standard._add_md5(config, pn, af)
570 611
@@ -574,6 +615,9 @@ def upgrade(args, config, basepath, workspace):
574 logger.info('New recipe is %s' % rf) 615 logger.info('New recipe is %s' % rf)
575 if license_diff: 616 if license_diff:
576 logger.info('License checksums have been updated in the new recipe; please refer to it for the difference between the old and the new license texts.') 617 logger.info('License checksums have been updated in the new recipe; please refer to it for the difference between the old and the new license texts.')
618 preferred_version = rd.getVar('PREFERRED_VERSION_%s' % rd.getVar('PN'))
619 if preferred_version:
620 logger.warning('Version is pinned to %s via PREFERRED_VERSION; it may need adjustment to match the new version before any further steps are taken' % preferred_version)
577 finally: 621 finally:
578 tinfoil.shutdown() 622 tinfoil.shutdown()
579 return 0 623 return 0
@@ -605,7 +649,7 @@ def check_upgrade_status(args, config, basepath, workspace):
605 for result in results: 649 for result in results:
606 # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason 650 # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason
607 if args.all or result[1] != 'MATCH': 651 if args.all or result[1] != 'MATCH':
608 logger.info("{:25} {:15} {:15} {} {} {}".format( result[0], 652 print("{:25} {:15} {:15} {} {} {}".format( result[0],
609 result[2], 653 result[2],
610 result[1] if result[1] != 'UPDATE' else (result[3] if not result[3].endswith("new-commits-available") else "new commits"), 654 result[1] if result[1] != 'UPDATE' else (result[3] if not result[3].endswith("new-commits-available") else "new commits"),
611 result[4], 655 result[4],
diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py
index e9d52bb67b..10945d6008 100644
--- a/scripts/lib/recipetool/append.py
+++ b/scripts/lib/recipetool/append.py
@@ -18,6 +18,7 @@ import shutil
18import scriptutils 18import scriptutils
19import errno 19import errno
20from collections import defaultdict 20from collections import defaultdict
21import difflib
21 22
22logger = logging.getLogger('recipetool') 23logger = logging.getLogger('recipetool')
23 24
@@ -49,7 +50,7 @@ def find_target_file(targetpath, d, pkglist=None):
49 '/etc/group': '/etc/group should be managed through the useradd and extrausers classes', 50 '/etc/group': '/etc/group should be managed through the useradd and extrausers classes',
50 '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes', 51 '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes',
51 '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes', 52 '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes',
52 '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname_pn-base-files = "value" in configuration',} 53 '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname:pn-base-files = "value" in configuration',}
53 54
54 for pthspec, message in invalidtargets.items(): 55 for pthspec, message in invalidtargets.items():
55 if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)): 56 if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)):
@@ -72,15 +73,15 @@ def find_target_file(targetpath, d, pkglist=None):
72 # This does assume that PN comes before other values, but that's a fairly safe assumption 73 # This does assume that PN comes before other values, but that's a fairly safe assumption
73 for line in f: 74 for line in f:
74 if line.startswith('PN:'): 75 if line.startswith('PN:'):
75 pn = line.split(':', 1)[1].strip() 76 pn = line.split(': ', 1)[1].strip()
76 elif line.startswith('FILES_INFO:'): 77 elif line.startswith('FILES_INFO'):
77 val = line.split(':', 1)[1].strip() 78 val = line.split(': ', 1)[1].strip()
78 dictval = json.loads(val) 79 dictval = json.loads(val)
79 for fullpth in dictval.keys(): 80 for fullpth in dictval.keys():
80 if fnmatch.fnmatchcase(fullpth, targetpath): 81 if fnmatch.fnmatchcase(fullpth, targetpath):
81 recipes[targetpath].append(pn) 82 recipes[targetpath].append(pn)
82 elif line.startswith('pkg_preinst_') or line.startswith('pkg_postinst_'): 83 elif line.startswith('pkg_preinst:') or line.startswith('pkg_postinst:'):
83 scriptval = line.split(':', 1)[1].strip().encode('utf-8').decode('unicode_escape') 84 scriptval = line.split(': ', 1)[1].strip().encode('utf-8').decode('unicode_escape')
84 if 'update-alternatives --install %s ' % targetpath in scriptval: 85 if 'update-alternatives --install %s ' % targetpath in scriptval:
85 recipes[targetpath].append('?%s' % pn) 86 recipes[targetpath].append('?%s' % pn)
86 elif targetpath_re.search(scriptval): 87 elif targetpath_re.search(scriptval):
@@ -100,7 +101,7 @@ def determine_file_source(targetpath, rd):
100 import oe.recipeutils 101 import oe.recipeutils
101 102
102 # See if it's in do_install for the recipe 103 # See if it's in do_install for the recipe
103 workdir = rd.getVar('WORKDIR') 104 unpackdir = rd.getVar('UNPACKDIR')
104 src_uri = rd.getVar('SRC_URI') 105 src_uri = rd.getVar('SRC_URI')
105 srcfile = '' 106 srcfile = ''
106 modpatches = [] 107 modpatches = []
@@ -112,9 +113,9 @@ def determine_file_source(targetpath, rd):
112 if not srcpath.startswith('/'): 113 if not srcpath.startswith('/'):
113 # Handle non-absolute path 114 # Handle non-absolute path
114 srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath)) 115 srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath))
115 if srcpath.startswith(workdir): 116 if srcpath.startswith(unpackdir):
116 # OK, now we have the source file name, look for it in SRC_URI 117 # OK, now we have the source file name, look for it in SRC_URI
117 workdirfile = os.path.relpath(srcpath, workdir) 118 workdirfile = os.path.relpath(srcpath, unpackdir)
118 # FIXME this is where we ought to have some code in the fetcher, because this is naive 119 # FIXME this is where we ought to have some code in the fetcher, because this is naive
119 for item in src_uri.split(): 120 for item in src_uri.split():
120 localpath = bb.fetch2.localpath(item, rd) 121 localpath = bb.fetch2.localpath(item, rd)
@@ -299,7 +300,10 @@ def appendfile(args):
299 if st.st_mode & stat.S_IXUSR: 300 if st.st_mode & stat.S_IXUSR:
300 perms = '0755' 301 perms = '0755'
301 install = {args.newfile: (args.targetpath, perms)} 302 install = {args.newfile: (args.targetpath, perms)}
302 oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine) 303 if sourcepath:
304 sourcepath = os.path.basename(sourcepath)
305 oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: {'newname' : sourcepath}}, install, wildcardver=args.wildcard_version, machine=args.machine)
306 tinfoil.modified_files()
303 return 0 307 return 0
304 else: 308 else:
305 if alternative_pns: 309 if alternative_pns:
@@ -327,6 +331,7 @@ def appendsrc(args, files, rd, extralines=None):
327 331
328 copyfiles = {} 332 copyfiles = {}
329 extralines = extralines or [] 333 extralines = extralines or []
334 params = []
330 for newfile, srcfile in files.items(): 335 for newfile, srcfile in files.items():
331 src_destdir = os.path.dirname(srcfile) 336 src_destdir = os.path.dirname(srcfile)
332 if not args.use_workdir: 337 if not args.use_workdir:
@@ -337,25 +342,46 @@ def appendsrc(args, files, rd, extralines=None):
337 src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir) 342 src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir)
338 src_destdir = os.path.normpath(src_destdir) 343 src_destdir = os.path.normpath(src_destdir)
339 344
340 source_uri = 'file://{0}'.format(os.path.basename(srcfile))
341 if src_destdir and src_destdir != '.': 345 if src_destdir and src_destdir != '.':
342 source_uri += ';subdir={0}'.format(src_destdir) 346 params.append({'subdir': src_destdir})
343
344 simple = bb.fetch.URI(source_uri)
345 simple.params = {}
346 simple_str = str(simple)
347 if simple_str in simplified:
348 existing = simplified[simple_str]
349 if source_uri != existing:
350 logger.warning('{0!r} is already in SRC_URI, with different parameters: {1!r}, not adding'.format(source_uri, existing))
351 else:
352 logger.warning('{0!r} is already in SRC_URI, not adding'.format(source_uri))
353 else: 347 else:
354 extralines.append('SRC_URI += {0}'.format(source_uri)) 348 params.append({})
355 copyfiles[newfile] = srcfile 349
356 350 copyfiles[newfile] = {'newname' : os.path.basename(srcfile)}
357 oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines) 351
358 352 dry_run_output = None
353 dry_run_outdir = None
354 if args.dry_run:
355 import tempfile
356 dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
357 dry_run_outdir = dry_run_output.name
358
359 appendfile, _ = oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines, params=params,
360 redirect_output=dry_run_outdir, update_original_recipe=args.update_recipe)
361 if not appendfile:
362 return
363 if args.dry_run:
364 output = ''
365 appendfilename = os.path.basename(appendfile)
366 newappendfile = appendfile
367 if appendfile and os.path.exists(appendfile):
368 with open(appendfile, 'r') as f:
369 oldlines = f.readlines()
370 else:
371 appendfile = '/dev/null'
372 oldlines = []
373
374 with open(os.path.join(dry_run_outdir, appendfilename), 'r') as f:
375 newlines = f.readlines()
376 diff = difflib.unified_diff(oldlines, newlines, appendfile, newappendfile)
377 difflines = list(diff)
378 if difflines:
379 output += ''.join(difflines)
380 if output:
381 logger.info('Diff of changed files:\n%s' % output)
382 else:
383 logger.info('No changed files')
384 tinfoil.modified_files()
359 385
360def appendsrcfiles(parser, args): 386def appendsrcfiles(parser, args):
361 recipedata = _parse_recipe(args.recipe, tinfoil) 387 recipedata = _parse_recipe(args.recipe, tinfoil)
@@ -435,6 +461,8 @@ def register_commands(subparsers):
435 help='Create/update a bbappend to add or replace source files', 461 help='Create/update a bbappend to add or replace source files',
436 description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.') 462 description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.')
437 parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path) 463 parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path)
464 parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true')
465 parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true')
438 parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path) 466 parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path)
439 parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True) 467 parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True)
440 468
@@ -442,6 +470,8 @@ def register_commands(subparsers):
442 parents=[common_src], 470 parents=[common_src],
443 help='Create/update a bbappend to add or replace a source file', 471 help='Create/update a bbappend to add or replace a source file',
444 description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.') 472 description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.')
473 parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true')
474 parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true')
445 parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path) 475 parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path)
446 parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path) 476 parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path)
447 parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True) 477 parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True)
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 566c75369a..8e9ff38db6 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -115,8 +115,8 @@ class RecipeHandler(object):
115 for line in f: 115 for line in f:
116 if line.startswith('PN:'): 116 if line.startswith('PN:'):
117 pn = line.split(':', 1)[-1].strip() 117 pn = line.split(':', 1)[-1].strip()
118 elif line.startswith('FILES_INFO:'): 118 elif line.startswith('FILES_INFO:%s:' % pkg):
119 val = line.split(':', 1)[1].strip() 119 val = line.split(': ', 1)[1].strip()
120 dictval = json.loads(val) 120 dictval = json.loads(val)
121 for fullpth in sorted(dictval): 121 for fullpth in sorted(dictval):
122 if fullpth.startswith(includedir) and fullpth.endswith('.h'): 122 if fullpth.startswith(includedir) and fullpth.endswith('.h'):
@@ -366,7 +366,7 @@ def supports_srcrev(uri):
366def reformat_git_uri(uri): 366def reformat_git_uri(uri):
367 '''Convert any http[s]://....git URI into git://...;protocol=http[s]''' 367 '''Convert any http[s]://....git URI into git://...;protocol=http[s]'''
368 checkuri = uri.split(';', 1)[0] 368 checkuri = uri.split(';', 1)[0]
369 if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri): 369 if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://git(hub|lab).com/[^/]+/[^/]+/?$', checkuri):
370 # Appends scheme if the scheme is missing 370 # Appends scheme if the scheme is missing
371 if not '://' in uri: 371 if not '://' in uri:
372 uri = 'git://' + uri 372 uri = 'git://' + uri
@@ -423,6 +423,36 @@ def create_recipe(args):
423 storeTagName = '' 423 storeTagName = ''
424 pv_srcpv = False 424 pv_srcpv = False
425 425
426 handled = []
427 classes = []
428
429 # Find all plugins that want to register handlers
430 logger.debug('Loading recipe handlers')
431 raw_handlers = []
432 for plugin in plugins:
433 if hasattr(plugin, 'register_recipe_handlers'):
434 plugin.register_recipe_handlers(raw_handlers)
435 # Sort handlers by priority
436 handlers = []
437 for i, handler in enumerate(raw_handlers):
438 if isinstance(handler, tuple):
439 handlers.append((handler[0], handler[1], i))
440 else:
441 handlers.append((handler, 0, i))
442 handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
443 for handler, priority, _ in handlers:
444 logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
445 setattr(handler, '_devtool', args.devtool)
446 handlers = [item[0] for item in handlers]
447
448 fetchuri = None
449 for handler in handlers:
450 if hasattr(handler, 'process_url'):
451 ret = handler.process_url(args, classes, handled, extravalues)
452 if 'url' in handled and ret:
453 fetchuri = ret
454 break
455
426 if os.path.isfile(source): 456 if os.path.isfile(source):
427 source = 'file://%s' % os.path.abspath(source) 457 source = 'file://%s' % os.path.abspath(source)
428 458
@@ -431,11 +461,12 @@ def create_recipe(args):
431 if re.match(r'https?://github.com/[^/]+/[^/]+/archive/.+(\.tar\..*|\.zip)$', source): 461 if re.match(r'https?://github.com/[^/]+/[^/]+/archive/.+(\.tar\..*|\.zip)$', source):
432 logger.warning('github archive files are not guaranteed to be stable and may be re-generated over time. If the latter occurs, the checksums will likely change and the recipe will fail at do_fetch. It is recommended that you point to an actual commit or tag in the repository instead (using the repository URL in conjunction with the -S/--srcrev option).') 462 logger.warning('github archive files are not guaranteed to be stable and may be re-generated over time. If the latter occurs, the checksums will likely change and the recipe will fail at do_fetch. It is recommended that you point to an actual commit or tag in the repository instead (using the repository URL in conjunction with the -S/--srcrev option).')
433 # Fetch a URL 463 # Fetch a URL
434 fetchuri = reformat_git_uri(urldefrag(source)[0]) 464 if not fetchuri:
465 fetchuri = reformat_git_uri(urldefrag(source)[0])
435 if args.binary: 466 if args.binary:
436 # Assume the archive contains the directory structure verbatim 467 # Assume the archive contains the directory structure verbatim
437 # so we need to extract to a subdirectory 468 # so we need to extract to a subdirectory
438 fetchuri += ';subdir=${BP}' 469 fetchuri += ';subdir=${BPN}'
439 srcuri = fetchuri 470 srcuri = fetchuri
440 rev_re = re.compile(';rev=([^;]+)') 471 rev_re = re.compile(';rev=([^;]+)')
441 res = rev_re.search(srcuri) 472 res = rev_re.search(srcuri)
@@ -478,6 +509,9 @@ def create_recipe(args):
478 storeTagName = params['tag'] 509 storeTagName = params['tag']
479 params['nobranch'] = '1' 510 params['nobranch'] = '1'
480 del params['tag'] 511 del params['tag']
512 # Assume 'master' branch if not set
513 if scheme in ['git', 'gitsm'] and 'branch' not in params and 'nobranch' not in params:
514 params['branch'] = 'master'
481 fetchuri = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params)) 515 fetchuri = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params))
482 516
483 tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR') 517 tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
@@ -527,10 +561,9 @@ def create_recipe(args):
527 # Remove HEAD reference point and drop remote prefix 561 # Remove HEAD reference point and drop remote prefix
528 get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')] 562 get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')]
529 if 'master' in get_branch: 563 if 'master' in get_branch:
530 # If it is master, we do not need to append 'branch=master' as this is default.
531 # Even with the case where get_branch has multiple objects, if 'master' is one 564 # Even with the case where get_branch has multiple objects, if 'master' is one
532 # of them, we should default take from 'master' 565 # of them, we should default take from 'master'
533 srcbranch = '' 566 srcbranch = 'master'
534 elif len(get_branch) == 1: 567 elif len(get_branch) == 1:
535 # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch' 568 # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch'
536 srcbranch = get_branch[0] 569 srcbranch = get_branch[0]
@@ -543,8 +576,8 @@ def create_recipe(args):
543 # Since we might have a value in srcbranch, we need to 576 # Since we might have a value in srcbranch, we need to
544 # recontruct the srcuri to include 'branch' in params. 577 # recontruct the srcuri to include 'branch' in params.
545 scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(srcuri) 578 scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(srcuri)
546 if srcbranch: 579 if scheme in ['git', 'gitsm']:
547 params['branch'] = srcbranch 580 params['branch'] = srcbranch or 'master'
548 581
549 if storeTagName and scheme in ['git', 'gitsm']: 582 if storeTagName and scheme in ['git', 'gitsm']:
550 # Check srcrev using tag and check validity of the tag 583 # Check srcrev using tag and check validity of the tag
@@ -603,7 +636,7 @@ def create_recipe(args):
603 splitline = line.split() 636 splitline = line.split()
604 if len(splitline) > 1: 637 if len(splitline) > 1:
605 if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]): 638 if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]):
606 srcuri = reformat_git_uri(splitline[1]) 639 srcuri = reformat_git_uri(splitline[1]) + ';branch=master'
607 srcsubdir = 'git' 640 srcsubdir = 'git'
608 break 641 break
609 642
@@ -636,8 +669,6 @@ def create_recipe(args):
636 # We'll come back and replace this later in handle_license_vars() 669 # We'll come back and replace this later in handle_license_vars()
637 lines_before.append('##LICENSE_PLACEHOLDER##') 670 lines_before.append('##LICENSE_PLACEHOLDER##')
638 671
639 handled = []
640 classes = []
641 672
642 # FIXME This is kind of a hack, we probably ought to be using bitbake to do this 673 # FIXME This is kind of a hack, we probably ought to be using bitbake to do this
643 pn = None 674 pn = None
@@ -675,8 +706,10 @@ def create_recipe(args):
675 if not srcuri: 706 if not srcuri:
676 lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') 707 lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
677 lines_before.append('SRC_URI = "%s"' % srcuri) 708 lines_before.append('SRC_URI = "%s"' % srcuri)
709 shown_checksums = ["%ssum" % s for s in bb.fetch2.SHOWN_CHECKSUM_LIST]
678 for key, value in sorted(checksums.items()): 710 for key, value in sorted(checksums.items()):
679 lines_before.append('SRC_URI[%s] = "%s"' % (key, value)) 711 if key in shown_checksums:
712 lines_before.append('SRC_URI[%s] = "%s"' % (key, value))
680 if srcuri and supports_srcrev(srcuri): 713 if srcuri and supports_srcrev(srcuri):
681 lines_before.append('') 714 lines_before.append('')
682 lines_before.append('# Modify these as desired') 715 lines_before.append('# Modify these as desired')
@@ -688,7 +721,7 @@ def create_recipe(args):
688 srcpvprefix = 'svnr' 721 srcpvprefix = 'svnr'
689 else: 722 else:
690 srcpvprefix = scheme 723 srcpvprefix = scheme
691 lines_before.append('PV = "%s+%s${SRCPV}"' % (realpv or '1.0', srcpvprefix)) 724 lines_before.append('PV = "%s+%s"' % (realpv or '1.0', srcpvprefix))
692 pv_srcpv = True 725 pv_srcpv = True
693 if not args.autorev and srcrev == '${AUTOREV}': 726 if not args.autorev and srcrev == '${AUTOREV}':
694 if os.path.exists(os.path.join(srctree, '.git')): 727 if os.path.exists(os.path.join(srctree, '.git')):
@@ -710,31 +743,12 @@ def create_recipe(args):
710 lines_after.append('') 743 lines_after.append('')
711 744
712 if args.binary: 745 if args.binary:
713 lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') 746 lines_after.append('INSANE_SKIP:${PN} += "already-stripped"')
714 lines_after.append('') 747 lines_after.append('')
715 748
716 if args.npm_dev: 749 if args.npm_dev:
717 extravalues['NPM_INSTALL_DEV'] = 1 750 extravalues['NPM_INSTALL_DEV'] = 1
718 751
719 # Find all plugins that want to register handlers
720 logger.debug('Loading recipe handlers')
721 raw_handlers = []
722 for plugin in plugins:
723 if hasattr(plugin, 'register_recipe_handlers'):
724 plugin.register_recipe_handlers(raw_handlers)
725 # Sort handlers by priority
726 handlers = []
727 for i, handler in enumerate(raw_handlers):
728 if isinstance(handler, tuple):
729 handlers.append((handler[0], handler[1], i))
730 else:
731 handlers.append((handler, 0, i))
732 handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
733 for handler, priority, _ in handlers:
734 logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
735 setattr(handler, '_devtool', args.devtool)
736 handlers = [item[0] for item in handlers]
737
738 # Apply the handlers 752 # Apply the handlers
739 if args.binary: 753 if args.binary:
740 classes.append('bin_package') 754 classes.append('bin_package')
@@ -743,6 +757,10 @@ def create_recipe(args):
743 for handler in handlers: 757 for handler in handlers:
744 handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues) 758 handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
745 759
760 # native and nativesdk classes are special and must be inherited last
761 # If present, put them at the end of the classes list
762 classes.sort(key=lambda c: c in ("native", "nativesdk"))
763
746 extrafiles = extravalues.pop('extrafiles', {}) 764 extrafiles = extravalues.pop('extrafiles', {})
747 extra_pn = extravalues.pop('PN', None) 765 extra_pn = extravalues.pop('PN', None)
748 extra_pv = extravalues.pop('PV', None) 766 extra_pv = extravalues.pop('PV', None)
@@ -867,8 +885,10 @@ def create_recipe(args):
867 outlines.append('') 885 outlines.append('')
868 outlines.extend(lines_after) 886 outlines.extend(lines_after)
869 887
888 outlines = [ line.rstrip('\n') +"\n" for line in outlines]
889
870 if extravalues: 890 if extravalues:
871 _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False) 891 _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=True)
872 892
873 if args.extract_to: 893 if args.extract_to:
874 scriptutils.git_convert_standalone_clone(srctree) 894 scriptutils.git_convert_standalone_clone(srctree)
@@ -884,7 +904,7 @@ def create_recipe(args):
884 log_info_cond('Source extracted to %s' % args.extract_to, args.devtool) 904 log_info_cond('Source extracted to %s' % args.extract_to, args.devtool)
885 905
886 if outfile == '-': 906 if outfile == '-':
887 sys.stdout.write('\n'.join(outlines) + '\n') 907 sys.stdout.write(''.join(outlines) + '\n')
888 else: 908 else:
889 with open(outfile, 'w') as f: 909 with open(outfile, 'w') as f:
890 lastline = None 910 lastline = None
@@ -892,9 +912,10 @@ def create_recipe(args):
892 if not lastline and not line: 912 if not lastline and not line:
893 # Skip extra blank lines 913 # Skip extra blank lines
894 continue 914 continue
895 f.write('%s\n' % line) 915 f.write('%s' % line)
896 lastline = line 916 lastline = line
897 log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool) 917 log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool)
918 tinfoil.modified_files()
898 919
899 if tempsrc: 920 if tempsrc:
900 if args.keep_temp: 921 if args.keep_temp:
@@ -917,6 +938,22 @@ def split_value(value):
917 else: 938 else:
918 return value 939 return value
919 940
941def fixup_license(value):
942 # Ensure licenses with OR starts and ends with brackets
943 if '|' in value:
944 return '(' + value + ')'
945 return value
946
947def tidy_licenses(value):
948 """Flat, split and sort licenses"""
949 from oe.license import flattened_licenses
950 def _choose(a, b):
951 str_a, str_b = sorted((" & ".join(a), " & ".join(b)), key=str.casefold)
952 return ["(%s | %s)" % (str_a, str_b)]
953 if not isinstance(value, str):
954 value = " & ".join(value)
955 return sorted(list(set(flattened_licenses(value, _choose))), key=str.casefold)
956
920def handle_license_vars(srctree, lines_before, handled, extravalues, d): 957def handle_license_vars(srctree, lines_before, handled, extravalues, d):
921 lichandled = [x for x in handled if x[0] == 'license'] 958 lichandled = [x for x in handled if x[0] == 'license']
922 if lichandled: 959 if lichandled:
@@ -930,10 +967,13 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
930 lines = [] 967 lines = []
931 if licvalues: 968 if licvalues:
932 for licvalue in licvalues: 969 for licvalue in licvalues:
933 if not licvalue[0] in licenses: 970 license = licvalue[0]
934 licenses.append(licvalue[0]) 971 lics = tidy_licenses(fixup_license(license))
972 lics = [lic for lic in lics if lic not in licenses]
973 if len(lics):
974 licenses.extend(lics)
935 lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) 975 lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
936 if licvalue[0] == 'Unknown': 976 if license == 'Unknown':
937 lic_unknown.append(licvalue[1]) 977 lic_unknown.append(licvalue[1])
938 if lic_unknown: 978 if lic_unknown:
939 lines.append('#') 979 lines.append('#')
@@ -942,9 +982,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
942 for licfile in lic_unknown: 982 for licfile in lic_unknown:
943 lines.append('# %s' % licfile) 983 lines.append('# %s' % licfile)
944 984
945 extra_license = split_value(extravalues.pop('LICENSE', [])) 985 extra_license = tidy_licenses(extravalues.pop('LICENSE', ''))
946 if '&' in extra_license:
947 extra_license.remove('&')
948 if extra_license: 986 if extra_license:
949 if licenses == ['Unknown']: 987 if licenses == ['Unknown']:
950 licenses = extra_license 988 licenses = extra_license
@@ -985,7 +1023,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
985 lines.append('# instead of &. If there is any doubt, check the accompanying documentation') 1023 lines.append('# instead of &. If there is any doubt, check the accompanying documentation')
986 lines.append('# to determine which situation is applicable.') 1024 lines.append('# to determine which situation is applicable.')
987 1025
988 lines.append('LICENSE = "%s"' % ' & '.join(licenses)) 1026 lines.append('LICENSE = "%s"' % ' & '.join(sorted(licenses, key=str.casefold)))
989 lines.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) 1027 lines.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum))
990 lines.append('') 1028 lines.append('')
991 1029
@@ -1002,118 +1040,170 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
1002 handled.append(('license', licvalues)) 1040 handled.append(('license', licvalues))
1003 return licvalues 1041 return licvalues
1004 1042
1005def get_license_md5sums(d, static_only=False): 1043def get_license_md5sums(d, static_only=False, linenumbers=False):
1006 import bb.utils 1044 import bb.utils
1045 import csv
1007 md5sums = {} 1046 md5sums = {}
1008 if not static_only: 1047 if not static_only and not linenumbers:
1009 # Gather md5sums of license files in common license dir 1048 # Gather md5sums of license files in common license dir
1010 commonlicdir = d.getVar('COMMON_LICENSE_DIR') 1049 commonlicdir = d.getVar('COMMON_LICENSE_DIR')
1011 for fn in os.listdir(commonlicdir): 1050 for fn in os.listdir(commonlicdir):
1012 md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn)) 1051 md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn))
1013 md5sums[md5value] = fn 1052 md5sums[md5value] = fn
1053
1014 # The following were extracted from common values in various recipes 1054 # The following were extracted from common values in various recipes
1015 # (double checking the license against the license file itself, not just 1055 # (double checking the license against the license file itself, not just
1016 # the LICENSE value in the recipe) 1056 # the LICENSE value in the recipe)
1017 md5sums['94d55d512a9ba36caa9b7df079bae19f'] = 'GPLv2' 1057
1018 md5sums['b234ee4d69f5fce4486a80fdaf4a4263'] = 'GPLv2' 1058 # Read license md5sums from csv file
1019 md5sums['59530bdf33659b29e73d4adb9f9f6552'] = 'GPLv2' 1059 scripts_path = os.path.dirname(os.path.realpath(__file__))
1020 md5sums['0636e73ff0215e8d672dc4c32c317bb3'] = 'GPLv2' 1060 for path in (d.getVar('BBPATH').split(':')
1021 md5sums['eb723b61539feef013de476e68b5c50a'] = 'GPLv2' 1061 + [os.path.join(scripts_path, '..', '..')]):
1022 md5sums['751419260aa954499f7abaabaa882bbe'] = 'GPLv2' 1062 csv_path = os.path.join(path, 'lib', 'recipetool', 'licenses.csv')
1023 md5sums['393a5ca445f6965873eca0259a17f833'] = 'GPLv2' 1063 if os.path.isfile(csv_path):
1024 md5sums['12f884d2ae1ff87c09e5b7ccc2c4ca7e'] = 'GPLv2' 1064 with open(csv_path, newline='') as csv_file:
1025 md5sums['8ca43cbc842c2336e835926c2166c28b'] = 'GPLv2' 1065 fieldnames = ['md5sum', 'license', 'beginline', 'endline', 'md5']
1026 md5sums['ebb5c50ab7cab4baeffba14977030c07'] = 'GPLv2' 1066 reader = csv.DictReader(csv_file, delimiter=',', fieldnames=fieldnames)
1027 md5sums['c93c0550bd3173f4504b2cbd8991e50b'] = 'GPLv2' 1067 for row in reader:
1028 md5sums['9ac2e7cff1ddaf48b6eab6028f23ef88'] = 'GPLv2' 1068 if linenumbers:
1029 md5sums['4325afd396febcb659c36b49533135d4'] = 'GPLv2' 1069 md5sums[row['md5sum']] = (
1030 md5sums['18810669f13b87348459e611d31ab760'] = 'GPLv2' 1070 row['license'], row['beginline'], row['endline'], row['md5'])
1031 md5sums['d7810fab7487fb0aad327b76f1be7cd7'] = 'GPLv2' # the Linux kernel's COPYING file 1071 else:
1032 md5sums['bbb461211a33b134d42ed5ee802b37ff'] = 'LGPLv2.1' 1072 md5sums[row['md5sum']] = row['license']
1033 md5sums['7fbc338309ac38fefcd64b04bb903e34'] = 'LGPLv2.1' 1073
1034 md5sums['4fbd65380cdd255951079008b364516c'] = 'LGPLv2.1'
1035 md5sums['2d5025d4aa3495befef8f17206a5b0a1'] = 'LGPLv2.1'
1036 md5sums['fbc093901857fcd118f065f900982c24'] = 'LGPLv2.1'
1037 md5sums['a6f89e2100d9b6cdffcea4f398e37343'] = 'LGPLv2.1'
1038 md5sums['d8045f3b8f929c1cb29a1e3fd737b499'] = 'LGPLv2.1'
1039 md5sums['fad9b3332be894bab9bc501572864b29'] = 'LGPLv2.1'
1040 md5sums['3bf50002aefd002f49e7bb854063f7e7'] = 'LGPLv2'
1041 md5sums['9f604d8a4f8e74f4f5140845a21b6674'] = 'LGPLv2'
1042 md5sums['5f30f0716dfdd0d91eb439ebec522ec2'] = 'LGPLv2'
1043 md5sums['55ca817ccb7d5b5b66355690e9abc605'] = 'LGPLv2'
1044 md5sums['252890d9eee26aab7b432e8b8a616475'] = 'LGPLv2'
1045 md5sums['3214f080875748938ba060314b4f727d'] = 'LGPLv2'
1046 md5sums['db979804f025cf55aabec7129cb671ed'] = 'LGPLv2'
1047 md5sums['d32239bcb673463ab874e80d47fae504'] = 'GPLv3'
1048 md5sums['f27defe1e96c2e1ecd4e0c9be8967949'] = 'GPLv3'
1049 md5sums['6a6a8e020838b23406c81b19c1d46df6'] = 'LGPLv3'
1050 md5sums['3b83ef96387f14655fc854ddc3c6bd57'] = 'Apache-2.0'
1051 md5sums['385c55653886acac3821999a3ccd17b3'] = 'Artistic-1.0 | GPL-2.0' # some perl modules
1052 md5sums['54c7042be62e169199200bc6477f04d1'] = 'BSD-3-Clause'
1053 md5sums['bfe1f75d606912a4111c90743d6c7325'] = 'MPL-1.1'
1054 return md5sums 1074 return md5sums
1055 1075
1056def crunch_license(licfile): 1076def crunch_known_licenses(d):
1057 ''' 1077 '''
1058 Remove non-material text from a license file and then check 1078 Calculate the MD5 checksums for the crunched versions of all common
1059 its md5sum against a known list. This works well for licenses 1079 licenses. Also add additional known checksums.
1060 which contain a copyright statement, but is also a useful way
1061 to handle people's insistence upon reformatting the license text
1062 slightly (with no material difference to the text of the
1063 license).
1064 ''' 1080 '''
1081
1082 crunched_md5sums = {}
1065 1083
1066 import oe.utils 1084 # common licenses
1085 crunched_md5sums['ad4e9d34a2e966dfe9837f18de03266d'] = 'GFDL-1.1-only'
1086 crunched_md5sums['d014fb11a34eb67dc717fdcfc97e60ed'] = 'GFDL-1.2-only'
1087 crunched_md5sums['e020ca655b06c112def28e597ab844f1'] = 'GFDL-1.3-only'
1067 1088
1068 # Note: these are carefully constructed!
1069 license_title_re = re.compile(r'^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$')
1070 license_statement_re = re.compile(r'^(This (project|software) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$')
1071 copyright_re = re.compile('^(#+)? *Copyright .*$')
1072
1073 crunched_md5sums = {}
1074 # The following two were gleaned from the "forever" npm package 1089 # The following two were gleaned from the "forever" npm package
1075 crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC' 1090 crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC'
1076 crunched_md5sums['eecf6429523cbc9693547cf2db790b5c'] = 'MIT'
1077 # https://github.com/vasi/pixz/blob/master/LICENSE
1078 crunched_md5sums['2f03392b40bbe663597b5bd3cc5ebdb9'] = 'BSD-2-Clause'
1079 # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt 1091 # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt
1080 crunched_md5sums['e72e5dfef0b1a4ca8a3d26a60587db66'] = 'BSD-2-Clause' 1092 crunched_md5sums['50fab24ce589d69af8964fdbfe414c60'] = 'BSD-2-Clause'
1081 # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE 1093 # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE
1082 crunched_md5sums['8be76ac6d191671f347ee4916baa637e'] = 'GPLv2' 1094 crunched_md5sums['88a4355858a1433fea99fae34a44da88'] = 'GPL-2.0-only'
1083 # https://github.com/datto/dattobd/blob/master/COPYING
1084 # http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/GPLv2.TXT
1085 crunched_md5sums['1d65c5ad4bf6489f85f4812bf08ae73d'] = 'GPLv2'
1086 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt 1095 # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt
1087 # http://git.neil.brown.name/?p=mdadm.git;a=blob;f=COPYING;h=d159169d1050894d3ea3b98e1c965c4058208fe1;hb=HEAD 1096 crunched_md5sums['063b5c3ebb5f3aa4c85a2ed18a31fbe7'] = 'GPL-2.0-only'
1088 crunched_md5sums['fb530f66a7a89ce920f0e912b5b66d4b'] = 'GPLv2'
1089 # https://github.com/gkos/nrf24/blob/master/COPYING
1090 crunched_md5sums['7b6aaa4daeafdfa6ed5443fd2684581b'] = 'GPLv2'
1091 # https://github.com/josch09/resetusb/blob/master/COPYING
1092 crunched_md5sums['8b8ac1d631a4d220342e83bcf1a1fbc3'] = 'GPLv3'
1093 # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1 1097 # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1
1094 crunched_md5sums['2ea316ed973ae176e502e2297b574bb3'] = 'LGPLv2.1' 1098 crunched_md5sums['7f5202f4d44ed15dcd4915f5210417d8'] = 'LGPL-2.1-only'
1095 # unixODBC-2.3.4 COPYING 1099 # unixODBC-2.3.4 COPYING
1096 crunched_md5sums['1daebd9491d1e8426900b4fa5a422814'] = 'LGPLv2.1' 1100 crunched_md5sums['3debde09238a8c8e1f6a847e1ec9055b'] = 'LGPL-2.1-only'
1097 # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3 1101 # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3
1098 crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3' 1102 crunched_md5sums['f90c613c51aa35da4d79dd55fc724ceb'] = 'LGPL-3.0-only'
1099 # https://raw.githubusercontent.com/eclipse/mosquitto/v1.4.14/epl-v10 1103 # https://raw.githubusercontent.com/eclipse/mosquitto/v1.4.14/epl-v10
1100 crunched_md5sums['efe2cb9a35826992b9df68224e3c2628'] = 'EPL-1.0' 1104 crunched_md5sums['efe2cb9a35826992b9df68224e3c2628'] = 'EPL-1.0'
1101 # https://raw.githubusercontent.com/eclipse/mosquitto/v1.4.14/edl-v10 1105
1102 crunched_md5sums['0a9c78c0a398d1bbce4a166757d60387'] = 'EDL-1.0' 1106 # https://raw.githubusercontent.com/jquery/esprima/3.1.3/LICENSE.BSD
1107 crunched_md5sums['80fa7b56a28e8c902e6af194003220a5'] = 'BSD-2-Clause'
1108 # https://raw.githubusercontent.com/npm/npm-install-checks/master/LICENSE
1109 crunched_md5sums['e659f77bfd9002659e112d0d3d59b2c1'] = 'BSD-2-Clause'
1110 # https://raw.githubusercontent.com/silverwind/default-gateway/4.2.0/LICENSE
1111 crunched_md5sums['4c641f2d995c47f5cb08bdb4b5b6ea05'] = 'BSD-2-Clause'
1112 # https://raw.githubusercontent.com/tad-lispy/node-damerau-levenshtein/v1.0.5/LICENSE
1113 crunched_md5sums['2b8c039b2b9a25f0feb4410c4542d346'] = 'BSD-2-Clause'
1114 # https://raw.githubusercontent.com/terser/terser/v3.17.0/LICENSE
1115 crunched_md5sums['8bd23871802951c9ad63855151204c2c'] = 'BSD-2-Clause'
1116 # https://raw.githubusercontent.com/alexei/sprintf.js/1.0.3/LICENSE
1117 crunched_md5sums['008c22318c8ea65928bf730ddd0273e3'] = 'BSD-3-Clause'
1118 # https://raw.githubusercontent.com/Caligatio/jsSHA/v3.2.0/LICENSE
1119 crunched_md5sums['0e46634a01bfef056892949acaea85b1'] = 'BSD-3-Clause'
1120 # https://raw.githubusercontent.com/d3/d3-path/v1.0.9/LICENSE
1121 crunched_md5sums['b5f72aef53d3b2b432702c30b0215666'] = 'BSD-3-Clause'
1122 # https://raw.githubusercontent.com/feross/ieee754/v1.1.13/LICENSE
1123 crunched_md5sums['a39327c997c20da0937955192d86232d'] = 'BSD-3-Clause'
1124 # https://raw.githubusercontent.com/joyent/node-extsprintf/v1.3.0/LICENSE
1125 crunched_md5sums['721f23a96ff4161ca3a5f071bbe18108'] = 'MIT'
1126 # https://raw.githubusercontent.com/pvorb/clone/v0.2.0/LICENSE
1127 crunched_md5sums['b376d29a53c9573006b9970709231431'] = 'MIT'
1128 # https://raw.githubusercontent.com/andris9/encoding/v0.1.12/LICENSE
1129 crunched_md5sums['85d8a977ee9d7c5ab4ac03c9b95431c4'] = 'MIT-0'
1130 # https://raw.githubusercontent.com/faye/websocket-driver-node/0.7.3/LICENSE.md
1131 crunched_md5sums['b66384e7137e41a9b1904ef4d39703b6'] = 'Apache-2.0'
1132 # https://raw.githubusercontent.com/less/less.js/v4.1.1/LICENSE
1133 crunched_md5sums['b27575459e02221ccef97ec0bfd457ae'] = 'Apache-2.0'
1134 # https://raw.githubusercontent.com/microsoft/TypeScript/v3.5.3/LICENSE.txt
1135 crunched_md5sums['a54a1a6a39e7f9dbb4a23a42f5c7fd1c'] = 'Apache-2.0'
1136 # https://raw.githubusercontent.com/request/request/v2.87.0/LICENSE
1137 crunched_md5sums['1034431802e57486b393d00c5d262b8a'] = 'Apache-2.0'
1138 # https://raw.githubusercontent.com/dchest/tweetnacl-js/v0.14.5/LICENSE
1139 crunched_md5sums['75605e6bdd564791ab698fca65c94a4f'] = 'Unlicense'
1140 # https://raw.githubusercontent.com/stackgl/gl-mat3/v2.0.0/LICENSE.md
1141 crunched_md5sums['75512892d6f59dddb6d1c7e191957e9c'] = 'Zlib'
1142
1143 commonlicdir = d.getVar('COMMON_LICENSE_DIR')
1144 for fn in sorted(os.listdir(commonlicdir)):
1145 md5value, lictext = crunch_license(os.path.join(commonlicdir, fn))
1146 if md5value not in crunched_md5sums:
1147 crunched_md5sums[md5value] = fn
1148 elif fn != crunched_md5sums[md5value]:
1149 bb.debug(2, "crunched_md5sums['%s'] is already set to '%s' rather than '%s'" % (md5value, crunched_md5sums[md5value], fn))
1150 else:
1151 bb.debug(2, "crunched_md5sums['%s'] is already set to '%s'" % (md5value, crunched_md5sums[md5value]))
1152
1153 return crunched_md5sums
1154
1155def crunch_license(licfile):
1156 '''
1157 Remove non-material text from a license file and then calculate its
1158 md5sum. This works well for licenses that contain a copyright statement,
1159 but is also a useful way to handle people's insistence upon reformatting
1160 the license text slightly (with no material difference to the text of the
1161 license).
1162 '''
1163
1164 import oe.utils
1165
1166 # Note: these are carefully constructed!
1167 license_title_re = re.compile(r'^#*\(? *(This is )?([Tt]he )?.{0,15} ?[Ll]icen[sc]e( \(.{1,10}\))?\)?[:\.]? ?#*$')
1168 license_statement_re = re.compile(r'^((This (project|software)|.{1,10}) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$')
1169 copyright_re = re.compile(r'^ *[#\*]* *(Modified work |MIT LICENSED )?Copyright ?(\([cC]\))? .*$')
1170 disclaimer_re = re.compile(r'^ *\*? ?All [Rr]ights [Rr]eserved\.$')
1171 email_re = re.compile(r'^.*<[\w\.-]*@[\w\.\-]*>$')
1172 header_re = re.compile(r'^(\/\**!?)? ?[\-=\*]* ?(\*\/)?$')
1173 tag_re = re.compile(r'^ *@?\(?([Ll]icense|MIT)\)?$')
1174 url_re = re.compile(r'^ *[#\*]* *https?:\/\/[\w\.\/\-]+$')
1175
1103 lictext = [] 1176 lictext = []
1104 with open(licfile, 'r', errors='surrogateescape') as f: 1177 with open(licfile, 'r', errors='surrogateescape') as f:
1105 for line in f: 1178 for line in f:
1106 # Drop opening statements 1179 # Drop opening statements
1107 if copyright_re.match(line): 1180 if copyright_re.match(line):
1108 continue 1181 continue
1182 elif disclaimer_re.match(line):
1183 continue
1184 elif email_re.match(line):
1185 continue
1186 elif header_re.match(line):
1187 continue
1188 elif tag_re.match(line):
1189 continue
1190 elif url_re.match(line):
1191 continue
1109 elif license_title_re.match(line): 1192 elif license_title_re.match(line):
1110 continue 1193 continue
1111 elif license_statement_re.match(line): 1194 elif license_statement_re.match(line):
1112 continue 1195 continue
1113 # Squash spaces, and replace smart quotes, double quotes 1196 # Strip comment symbols
1114 # and backticks with single quotes 1197 line = line.replace('*', '') \
1198 .replace('#', '')
1199 # Unify spelling
1200 line = line.replace('sub-license', 'sublicense')
1201 # Squash spaces
1115 line = oe.utils.squashspaces(line.strip()) 1202 line = oe.utils.squashspaces(line.strip())
1203 # Replace smart quotes, double quotes and backticks with single quotes
1116 line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'') 1204 line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'')
1205 # Unify brackets
1206 line = line.replace("{", "[").replace("}", "]")
1117 if line: 1207 if line:
1118 lictext.append(line) 1208 lictext.append(line)
1119 1209
@@ -1124,31 +1214,40 @@ def crunch_license(licfile):
1124 except UnicodeEncodeError: 1214 except UnicodeEncodeError:
1125 md5val = None 1215 md5val = None
1126 lictext = '' 1216 lictext = ''
1127 license = crunched_md5sums.get(md5val, None) 1217 return md5val, lictext
1128 return license, md5val, lictext
1129 1218
1130def guess_license(srctree, d): 1219def guess_license(srctree, d):
1131 import bb 1220 import bb
1132 md5sums = get_license_md5sums(d) 1221 md5sums = get_license_md5sums(d)
1133 1222
1223 crunched_md5sums = crunch_known_licenses(d)
1224
1134 licenses = [] 1225 licenses = []
1135 licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*', 'e[dp]l-v10'] 1226 licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*', 'e[dp]l-v10']
1227 skip_extensions = (".html", ".js", ".json", ".svg", ".ts", ".go")
1136 licfiles = [] 1228 licfiles = []
1137 for root, dirs, files in os.walk(srctree): 1229 for root, dirs, files in os.walk(srctree):
1138 for fn in files: 1230 for fn in files:
1231 if fn.endswith(skip_extensions):
1232 continue
1139 for spec in licspecs: 1233 for spec in licspecs:
1140 if fnmatch.fnmatch(fn, spec): 1234 if fnmatch.fnmatch(fn, spec):
1141 fullpath = os.path.join(root, fn) 1235 fullpath = os.path.join(root, fn)
1142 if not fullpath in licfiles: 1236 if not fullpath in licfiles:
1143 licfiles.append(fullpath) 1237 licfiles.append(fullpath)
1144 for licfile in licfiles: 1238 for licfile in sorted(licfiles):
1145 md5value = bb.utils.md5_file(licfile) 1239 md5value = bb.utils.md5_file(licfile)
1146 license = md5sums.get(md5value, None) 1240 license = md5sums.get(md5value, None)
1147 if not license: 1241 if not license:
1148 license, crunched_md5, lictext = crunch_license(licfile) 1242 crunched_md5, lictext = crunch_license(licfile)
1149 if not license: 1243 license = crunched_md5sums.get(crunched_md5, None)
1244 if lictext and not license:
1150 license = 'Unknown' 1245 license = 'Unknown'
1151 licenses.append((license, os.path.relpath(licfile, srctree), md5value)) 1246 logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
1247 "and replace `Unknown` with the license:\n" \
1248 "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
1249 if license:
1250 licenses.append((license, os.path.relpath(licfile, srctree), md5value))
1152 1251
1153 # FIXME should we grab at least one source file with a license header and add that too? 1252 # FIXME should we grab at least one source file with a license header and add that too?
1154 1253
@@ -1162,6 +1261,7 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn
1162 """ 1261 """
1163 pkglicenses = {pn: []} 1262 pkglicenses = {pn: []}
1164 for license, licpath, _ in licvalues: 1263 for license, licpath, _ in licvalues:
1264 license = fixup_license(license)
1165 for pkgname, pkgpath in packages.items(): 1265 for pkgname, pkgpath in packages.items():
1166 if licpath.startswith(pkgpath + '/'): 1266 if licpath.startswith(pkgpath + '/'):
1167 if pkgname in pkglicenses: 1267 if pkgname in pkglicenses:
@@ -1174,11 +1274,14 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn
1174 pkglicenses[pn].append(license) 1274 pkglicenses[pn].append(license)
1175 outlicenses = {} 1275 outlicenses = {}
1176 for pkgname in packages: 1276 for pkgname in packages:
1177 license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown' 1277 # Assume AND operator between license files
1178 if license == 'Unknown' and pkgname in fallback_licenses: 1278 license = ' & '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown'
1279 if license == 'Unknown' and fallback_licenses and pkgname in fallback_licenses:
1179 license = fallback_licenses[pkgname] 1280 license = fallback_licenses[pkgname]
1180 outlines.append('LICENSE_%s = "%s"' % (pkgname, license)) 1281 licenses = tidy_licenses(license)
1181 outlicenses[pkgname] = license.split() 1282 license = ' & '.join(licenses)
1283 outlines.append('LICENSE:%s = "%s"' % (pkgname, license))
1284 outlicenses[pkgname] = licenses
1182 return outlicenses 1285 return outlicenses
1183 1286
1184def read_pkgconfig_provides(d): 1287def read_pkgconfig_provides(d):
@@ -1311,6 +1414,7 @@ def register_commands(subparsers):
1311 parser_create.add_argument('-B', '--srcbranch', help='Branch in source repository if fetching from an SCM such as git (default master)') 1414 parser_create.add_argument('-B', '--srcbranch', help='Branch in source repository if fetching from an SCM such as git (default master)')
1312 parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)') 1415 parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
1313 parser_create.add_argument('--npm-dev', action="store_true", help='For npm, also fetch devDependencies') 1416 parser_create.add_argument('--npm-dev', action="store_true", help='For npm, also fetch devDependencies')
1417 parser_create.add_argument('--no-pypi', action="store_true", help='Do not inherit pypi class')
1314 parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS) 1418 parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS)
1315 parser_create.add_argument('--mirrors', action="store_true", help='Enable PREMIRRORS and MIRRORS for source tree fetching (disabled by default).') 1419 parser_create.add_argument('--mirrors', action="store_true", help='Enable PREMIRRORS and MIRRORS for source tree fetching (disabled by default).')
1316 parser_create.set_defaults(func=create_recipe) 1420 parser_create.set_defaults(func=create_recipe)
diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py
index 35a97c9345..ec9d510e23 100644
--- a/scripts/lib/recipetool/create_buildsys.py
+++ b/scripts/lib/recipetool/create_buildsys.py
@@ -5,9 +5,9 @@
5# SPDX-License-Identifier: GPL-2.0-only 5# SPDX-License-Identifier: GPL-2.0-only
6# 6#
7 7
8import os
8import re 9import re
9import logging 10import logging
10import glob
11from recipetool.create import RecipeHandler, validate_pv 11from recipetool.create import RecipeHandler, validate_pv
12 12
13logger = logging.getLogger('recipetool') 13logger = logging.getLogger('recipetool')
@@ -137,15 +137,15 @@ class CmakeRecipeHandler(RecipeHandler):
137 deps = [] 137 deps = []
138 unmappedpkgs = [] 138 unmappedpkgs = []
139 139
140 proj_re = re.compile('project\s*\(([^)]*)\)', re.IGNORECASE) 140 proj_re = re.compile(r'project\s*\(([^)]*)\)', re.IGNORECASE)
141 pkgcm_re = re.compile('pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE) 141 pkgcm_re = re.compile(r'pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE)
142 pkgsm_re = re.compile('pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE) 142 pkgsm_re = re.compile(r'pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE)
143 findpackage_re = re.compile('find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE) 143 findpackage_re = re.compile(r'find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE)
144 findlibrary_re = re.compile('find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*') 144 findlibrary_re = re.compile(r'find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*')
145 checklib_re = re.compile('check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE) 145 checklib_re = re.compile(r'check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE)
146 include_re = re.compile('include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE) 146 include_re = re.compile(r'include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE)
147 subdir_re = re.compile('add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE) 147 subdir_re = re.compile(r'add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE)
148 dep_re = re.compile('([^ ><=]+)( *[<>=]+ *[^ ><=]+)?') 148 dep_re = re.compile(r'([^ ><=]+)( *[<>=]+ *[^ ><=]+)?')
149 149
150 def find_cmake_package(pkg): 150 def find_cmake_package(pkg):
151 RecipeHandler.load_devel_filemap(tinfoil.config_data) 151 RecipeHandler.load_devel_filemap(tinfoil.config_data)
@@ -423,16 +423,16 @@ class AutotoolsRecipeHandler(RecipeHandler):
423 'makeinfo': 'texinfo', 423 'makeinfo': 'texinfo',
424 } 424 }
425 425
426 pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') 426 pkg_re = re.compile(r'PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
427 pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*') 427 pkgce_re = re.compile(r'PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*')
428 lib_re = re.compile('AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*') 428 lib_re = re.compile(r'AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*')
429 libx_re = re.compile('AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*') 429 libx_re = re.compile(r'AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*')
430 progs_re = re.compile('_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*') 430 progs_re = re.compile(r'_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
431 dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?') 431 dep_re = re.compile(r'([^ ><=]+)( [<>=]+ [^ ><=]+)?')
432 ac_init_re = re.compile('AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*') 432 ac_init_re = re.compile(r'AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*')
433 am_init_re = re.compile('AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*') 433 am_init_re = re.compile(r'AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*')
434 define_re = re.compile('\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)') 434 define_re = re.compile(r'\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)')
435 version_re = re.compile('([0-9.]+)') 435 version_re = re.compile(r'([0-9.]+)')
436 436
437 defines = {} 437 defines = {}
438 def subst_defines(value): 438 def subst_defines(value):
@@ -545,7 +545,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
545 deps.append('zlib') 545 deps.append('zlib')
546 elif keyword in ('AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO'): 546 elif keyword in ('AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO'):
547 deps.append('openssl') 547 deps.append('openssl')
548 elif keyword == 'AX_LIB_CURL': 548 elif keyword in ('AX_LIB_CURL', 'LIBCURL_CHECK_CONFIG'):
549 deps.append('curl') 549 deps.append('curl')
550 elif keyword == 'AX_LIB_BEECRYPT': 550 elif keyword == 'AX_LIB_BEECRYPT':
551 deps.append('beecrypt') 551 deps.append('beecrypt')
@@ -624,6 +624,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
624 'AX_CHECK_OPENSSL', 624 'AX_CHECK_OPENSSL',
625 'AX_LIB_CRYPTO', 625 'AX_LIB_CRYPTO',
626 'AX_LIB_CURL', 626 'AX_LIB_CURL',
627 'LIBCURL_CHECK_CONFIG',
627 'AX_LIB_BEECRYPT', 628 'AX_LIB_BEECRYPT',
628 'AX_LIB_EXPAT', 629 'AX_LIB_EXPAT',
629 'AX_LIB_GCRYPT', 630 'AX_LIB_GCRYPT',
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py
index adfa377956..a807dafae5 100644
--- a/scripts/lib/recipetool/create_buildsys_python.py
+++ b/scripts/lib/recipetool/create_buildsys_python.py
@@ -8,9 +8,9 @@
8import ast 8import ast
9import codecs 9import codecs
10import collections 10import collections
11import distutils.command.build_py 11import setuptools.command.build_py
12import email 12import email
13import imp 13import importlib
14import glob 14import glob
15import itertools 15import itertools
16import logging 16import logging
@@ -18,7 +18,11 @@ import os
18import re 18import re
19import sys 19import sys
20import subprocess 20import subprocess
21import json
22import urllib.request
21from recipetool.create import RecipeHandler 23from recipetool.create import RecipeHandler
24from urllib.parse import urldefrag
25from recipetool.create import determine_from_url
22 26
23logger = logging.getLogger('recipetool') 27logger = logging.getLogger('recipetool')
24 28
@@ -37,7 +41,334 @@ class PythonRecipeHandler(RecipeHandler):
37 assume_provided = ['builtins', 'os.path'] 41 assume_provided = ['builtins', 'os.path']
38 # Assumes that the host python3 builtin_module_names is sane for target too 42 # Assumes that the host python3 builtin_module_names is sane for target too
39 assume_provided = assume_provided + list(sys.builtin_module_names) 43 assume_provided = assume_provided + list(sys.builtin_module_names)
44 excluded_fields = []
40 45
46
47 classifier_license_map = {
48 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
49 'License :: OSI Approved :: Apache Software License': 'Apache',
50 'License :: OSI Approved :: Apple Public Source License': 'APSL',
51 'License :: OSI Approved :: Artistic License': 'Artistic',
52 'License :: OSI Approved :: Attribution Assurance License': 'AAL',
53 'License :: OSI Approved :: BSD License': 'BSD-3-Clause',
54 'License :: OSI Approved :: Boost Software License 1.0 (BSL-1.0)': 'BSL-1.0',
55 'License :: OSI Approved :: CEA CNRS Inria Logiciel Libre License, version 2.1 (CeCILL-2.1)': 'CECILL-2.1',
56 'License :: OSI Approved :: Common Development and Distribution License 1.0 (CDDL-1.0)': 'CDDL-1.0',
57 'License :: OSI Approved :: Common Public License': 'CPL',
58 'License :: OSI Approved :: Eclipse Public License 1.0 (EPL-1.0)': 'EPL-1.0',
59 'License :: OSI Approved :: Eclipse Public License 2.0 (EPL-2.0)': 'EPL-2.0',
60 'License :: OSI Approved :: Eiffel Forum License': 'EFL',
61 'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0',
62 'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1',
63 'License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)': 'EUPL-1.2',
64 'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0-only',
65 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0-or-later',
66 'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL',
67 'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL',
68 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0-only',
69 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0-or-later',
70 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0-only',
71 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0-or-later',
72 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0-only',
73 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0-or-later',
74 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0-only',
75 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0-or-later',
76 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL',
77 'License :: OSI Approved :: Historical Permission Notice and Disclaimer (HPND)': 'HPND',
78 'License :: OSI Approved :: IBM Public License': 'IPL',
79 'License :: OSI Approved :: ISC License (ISCL)': 'ISC',
80 'License :: OSI Approved :: Intel Open Source License': 'Intel',
81 'License :: OSI Approved :: Jabber Open Source License': 'Jabber',
82 'License :: OSI Approved :: MIT License': 'MIT',
83 'License :: OSI Approved :: MIT No Attribution License (MIT-0)': 'MIT-0',
84 'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL',
85 'License :: OSI Approved :: MirOS License (MirOS)': 'MirOS',
86 'License :: OSI Approved :: Motosoto License': 'Motosoto',
87 'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0',
88 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1',
89 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0',
90 'License :: OSI Approved :: Nethack General Public License': 'NGPL',
91 'License :: OSI Approved :: Nokia Open Source License': 'Nokia',
92 'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL',
93 'License :: OSI Approved :: Open Software License 3.0 (OSL-3.0)': 'OSL-3.0',
94 'License :: OSI Approved :: PostgreSQL License': 'PostgreSQL',
95 'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python',
96 'License :: OSI Approved :: Python Software Foundation License': 'PSF-2.0',
97 'License :: OSI Approved :: Qt Public License (QPL)': 'QPL',
98 'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL',
99 'License :: OSI Approved :: SIL Open Font License 1.1 (OFL-1.1)': 'OFL-1.1',
100 'License :: OSI Approved :: Sleepycat License': 'Sleepycat',
101 'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': 'SISSL',
102 'License :: OSI Approved :: Sun Public License': 'SPL',
103 'License :: OSI Approved :: The Unlicense (Unlicense)': 'Unlicense',
104 'License :: OSI Approved :: Universal Permissive License (UPL)': 'UPL-1.0',
105 'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA',
106 'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0',
107 'License :: OSI Approved :: W3C License': 'W3C',
108 'License :: OSI Approved :: X.Net License': 'Xnet',
109 'License :: OSI Approved :: Zope Public License': 'ZPL',
110 'License :: OSI Approved :: zlib/libpng License': 'Zlib',
111 'License :: Other/Proprietary License': 'Proprietary',
112 'License :: Public Domain': 'PD',
113 }
114
115 def __init__(self):
116 pass
117
118 def process_url(self, args, classes, handled, extravalues):
119 """
120 Convert any pypi url https://pypi.org/project/<package>/<version> into https://files.pythonhosted.org/packages/source/...
121 which corresponds to the archive location, and add pypi class
122 """
123
124 if 'url' in handled:
125 return None
126
127 fetch_uri = None
128 source = args.source
129 required_version = args.version if args.version else None
130 match = re.match(r'https?://pypi.org/project/([^/]+)(?:/([^/]+))?/?$', urldefrag(source)[0])
131 if match:
132 package = match.group(1)
133 version = match.group(2) if match.group(2) else required_version
134
135 json_url = f"https://pypi.org/pypi/%s/json" % package
136 response = urllib.request.urlopen(json_url)
137 if response.status == 200:
138 data = json.loads(response.read())
139 if not version:
140 # grab latest version
141 version = data["info"]["version"]
142 pypi_package = data["info"]["name"]
143 for release in reversed(data["releases"][version]):
144 if release["packagetype"] == "sdist":
145 fetch_uri = release["url"]
146 break
147 else:
148 logger.warning("Cannot handle pypi url %s: cannot fetch package information using %s", source, json_url)
149 return None
150 else:
151 match = re.match(r'^https?://files.pythonhosted.org/packages.*/(.*)-.*$', source)
152 if match:
153 fetch_uri = source
154 pypi_package = match.group(1)
155 _, version = determine_from_url(fetch_uri)
156
157 if match and not args.no_pypi:
158 if required_version and version != required_version:
159 raise Exception("Version specified using --version/-V (%s) and version specified in the url (%s) do not match" % (required_version, version))
160 # This is optionnal if BPN looks like "python-<pypi_package>" or "python3-<pypi_package>" (see pypi.bbclass)
161 # but at this point we cannot know because because user can specify the output name of the recipe on the command line
162 extravalues["PYPI_PACKAGE"] = pypi_package
163 # If the tarball extension is not 'tar.gz' (default value in pypi.bblcass) whe should set PYPI_PACKAGE_EXT in the recipe
164 pypi_package_ext = re.match(r'.*%s-%s\.(.*)$' % (pypi_package, version), fetch_uri)
165 if pypi_package_ext:
166 pypi_package_ext = pypi_package_ext.group(1)
167 if pypi_package_ext != "tar.gz":
168 extravalues["PYPI_PACKAGE_EXT"] = pypi_package_ext
169
170 # Pypi class will handle S and SRC_URI variables, so remove them
171 # TODO: allow oe.recipeutils.patch_recipe_lines() to accept regexp so we can simplify the following to:
172 # extravalues['SRC_URI(?:\[.*?\])?'] = None
173 extravalues['S'] = None
174 extravalues['SRC_URI'] = None
175
176 classes.append('pypi')
177
178 handled.append('url')
179 return fetch_uri
180
181 def handle_classifier_license(self, classifiers, existing_licenses=""):
182
183 licenses = []
184 for classifier in classifiers:
185 if classifier in self.classifier_license_map:
186 license = self.classifier_license_map[classifier]
187 if license == 'Apache' and 'Apache-2.0' in existing_licenses:
188 license = 'Apache-2.0'
189 elif license == 'GPL':
190 if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
191 license = 'GPL-2.0'
192 elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
193 license = 'GPL-3.0'
194 elif license == 'LGPL':
195 if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
196 license = 'LGPL-2.1'
197 elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
198 license = 'LGPL-2.0'
199 elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
200 license = 'LGPL-3.0'
201 licenses.append(license)
202
203 if licenses:
204 return ' & '.join(licenses)
205
206 return None
207
208 def map_info_to_bbvar(self, info, extravalues):
209
210 # Map PKG-INFO & setup.py fields to bitbake variables
211 for field, values in info.items():
212 if field in self.excluded_fields:
213 continue
214
215 if field not in self.bbvar_map:
216 continue
217
218 if isinstance(values, str):
219 value = values
220 else:
221 value = ' '.join(str(v) for v in values if v)
222
223 bbvar = self.bbvar_map[field]
224 if bbvar == "PN":
225 # by convention python recipes start with "python3-"
226 if not value.startswith('python'):
227 value = 'python3-' + value
228
229 if bbvar not in extravalues and value:
230 extravalues[bbvar] = value
231
232 def apply_info_replacements(self, info):
233 if not self.replacements:
234 return
235
236 for variable, search, replace in self.replacements:
237 if variable not in info:
238 continue
239
240 def replace_value(search, replace, value):
241 if replace is None:
242 if re.search(search, value):
243 return None
244 else:
245 new_value = re.sub(search, replace, value)
246 if value != new_value:
247 return new_value
248 return value
249
250 value = info[variable]
251 if isinstance(value, str):
252 new_value = replace_value(search, replace, value)
253 if new_value is None:
254 del info[variable]
255 elif new_value != value:
256 info[variable] = new_value
257 elif hasattr(value, 'items'):
258 for dkey, dvalue in list(value.items()):
259 new_list = []
260 for pos, a_value in enumerate(dvalue):
261 new_value = replace_value(search, replace, a_value)
262 if new_value is not None and new_value != value:
263 new_list.append(new_value)
264
265 if value != new_list:
266 value[dkey] = new_list
267 else:
268 new_list = []
269 for pos, a_value in enumerate(value):
270 new_value = replace_value(search, replace, a_value)
271 if new_value is not None and new_value != value:
272 new_list.append(new_value)
273
274 if value != new_list:
275 info[variable] = new_list
276
277
278 def scan_python_dependencies(self, paths):
279 deps = set()
280 try:
281 dep_output = self.run_command(['pythondeps', '-d'] + paths)
282 except (OSError, subprocess.CalledProcessError):
283 pass
284 else:
285 for line in dep_output.splitlines():
286 line = line.rstrip()
287 dep, filename = line.split('\t', 1)
288 if filename.endswith('/setup.py'):
289 continue
290 deps.add(dep)
291
292 try:
293 provides_output = self.run_command(['pythondeps', '-p'] + paths)
294 except (OSError, subprocess.CalledProcessError):
295 pass
296 else:
297 provides_lines = (l.rstrip() for l in provides_output.splitlines())
298 provides = set(l for l in provides_lines if l and l != 'setup')
299 deps -= provides
300
301 return deps
302
303 def parse_pkgdata_for_python_packages(self):
304 pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
305
306 ldata = tinfoil.config_data.createCopy()
307 bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True)
308 python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
309
310 dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
311 python_dirs = [python_sitedir + os.sep,
312 os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
313 os.path.dirname(python_sitedir) + os.sep]
314 packages = {}
315 for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
316 files_info = None
317 with open(pkgdatafile, 'r') as f:
318 for line in f.readlines():
319 field, value = line.split(': ', 1)
320 if field.startswith('FILES_INFO'):
321 files_info = ast.literal_eval(value)
322 break
323 else:
324 continue
325
326 for fn in files_info:
327 for suffix in importlib.machinery.all_suffixes():
328 if fn.endswith(suffix):
329 break
330 else:
331 continue
332
333 if fn.startswith(dynload_dir + os.sep):
334 if '/.debug/' in fn:
335 continue
336 base = os.path.basename(fn)
337 provided = base.split('.', 1)[0]
338 packages[provided] = os.path.basename(pkgdatafile)
339 continue
340
341 for python_dir in python_dirs:
342 if fn.startswith(python_dir):
343 relpath = fn[len(python_dir):]
344 relstart, _, relremaining = relpath.partition(os.sep)
345 if relstart.endswith('.egg'):
346 relpath = relremaining
347 base, _ = os.path.splitext(relpath)
348
349 if '/.debug/' in base:
350 continue
351 if os.path.basename(base) == '__init__':
352 base = os.path.dirname(base)
353 base = base.replace(os.sep + os.sep, os.sep)
354 provided = base.replace(os.sep, '.')
355 packages[provided] = os.path.basename(pkgdatafile)
356 return packages
357
358 @classmethod
359 def run_command(cls, cmd, **popenargs):
360 if 'stderr' not in popenargs:
361 popenargs['stderr'] = subprocess.STDOUT
362 try:
363 return subprocess.check_output(cmd, **popenargs).decode('utf-8')
364 except OSError as exc:
365 logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
366 raise
367 except subprocess.CalledProcessError as exc:
368 logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
369 raise
370
371class PythonSetupPyRecipeHandler(PythonRecipeHandler):
41 bbvar_map = { 372 bbvar_map = {
42 'Name': 'PN', 373 'Name': 'PN',
43 'Version': 'PV', 374 'Version': 'PV',
@@ -45,9 +376,9 @@ class PythonRecipeHandler(RecipeHandler):
45 'Summary': 'SUMMARY', 376 'Summary': 'SUMMARY',
46 'Description': 'DESCRIPTION', 377 'Description': 'DESCRIPTION',
47 'License': 'LICENSE', 378 'License': 'LICENSE',
48 'Requires': 'RDEPENDS_${PN}', 379 'Requires': 'RDEPENDS:${PN}',
49 'Provides': 'RPROVIDES_${PN}', 380 'Provides': 'RPROVIDES:${PN}',
50 'Obsoletes': 'RREPLACES_${PN}', 381 'Obsoletes': 'RREPLACES:${PN}',
51 } 382 }
52 # PN/PV are already set by recipetool core & desc can be extremely long 383 # PN/PV are already set by recipetool core & desc can be extremely long
53 excluded_fields = [ 384 excluded_fields = [
@@ -75,6 +406,7 @@ class PythonRecipeHandler(RecipeHandler):
75 'Supported-Platform', 406 'Supported-Platform',
76 ] 407 ]
77 setuparg_multi_line_values = ['Description'] 408 setuparg_multi_line_values = ['Description']
409
78 replacements = [ 410 replacements = [
79 ('License', r' +$', ''), 411 ('License', r' +$', ''),
80 ('License', r'^ +', ''), 412 ('License', r'^ +', ''),
@@ -95,71 +427,161 @@ class PythonRecipeHandler(RecipeHandler):
95 ('Install-requires', r'\[[^\]]+\]$', ''), 427 ('Install-requires', r'\[[^\]]+\]$', ''),
96 ] 428 ]
97 429
98 classifier_license_map = {
99 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
100 'License :: OSI Approved :: Apache Software License': 'Apache',
101 'License :: OSI Approved :: Apple Public Source License': 'APSL',
102 'License :: OSI Approved :: Artistic License': 'Artistic',
103 'License :: OSI Approved :: Attribution Assurance License': 'AAL',
104 'License :: OSI Approved :: BSD License': 'BSD',
105 'License :: OSI Approved :: Common Public License': 'CPL',
106 'License :: OSI Approved :: Eiffel Forum License': 'EFL',
107 'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0',
108 'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1',
109 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0+',
110 'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0',
111 'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL',
112 'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL',
113 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0',
114 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0+',
115 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0',
116 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0+',
117 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0',
118 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0+',
119 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0',
120 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0+',
121 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL',
122 'License :: OSI Approved :: IBM Public License': 'IPL',
123 'License :: OSI Approved :: ISC License (ISCL)': 'ISC',
124 'License :: OSI Approved :: Intel Open Source License': 'Intel',
125 'License :: OSI Approved :: Jabber Open Source License': 'Jabber',
126 'License :: OSI Approved :: MIT License': 'MIT',
127 'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL',
128 'License :: OSI Approved :: Motosoto License': 'Motosoto',
129 'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0',
130 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1',
131 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0',
132 'License :: OSI Approved :: Nethack General Public License': 'NGPL',
133 'License :: OSI Approved :: Nokia Open Source License': 'Nokia',
134 'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL',
135 'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python',
136 'License :: OSI Approved :: Python Software Foundation License': 'PSF',
137 'License :: OSI Approved :: Qt Public License (QPL)': 'QPL',
138 'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL',
139 'License :: OSI Approved :: Sleepycat License': 'Sleepycat',
140 'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': '-- Sun Industry Standards Source License (SISSL)',
141 'License :: OSI Approved :: Sun Public License': 'SPL',
142 'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA',
143 'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0',
144 'License :: OSI Approved :: W3C License': 'W3C',
145 'License :: OSI Approved :: X.Net License': 'Xnet',
146 'License :: OSI Approved :: Zope Public License': 'ZPL',
147 'License :: OSI Approved :: zlib/libpng License': 'Zlib',
148 }
149
150 def __init__(self): 430 def __init__(self):
151 pass 431 pass
152 432
433 def parse_setup_py(self, setupscript='./setup.py'):
434 with codecs.open(setupscript) as f:
435 info, imported_modules, non_literals, extensions = gather_setup_info(f)
436
437 def _map(key):
438 key = key.replace('_', '-')
439 key = key[0].upper() + key[1:]
440 if key in self.setup_parse_map:
441 key = self.setup_parse_map[key]
442 return key
443
444 # Naive mapping of setup() arguments to PKG-INFO field names
445 for d in [info, non_literals]:
446 for key, value in list(d.items()):
447 if key is None:
448 continue
449 new_key = _map(key)
450 if new_key != key:
451 del d[key]
452 d[new_key] = value
453
454 return info, 'setuptools' in imported_modules, non_literals, extensions
455
456 def get_setup_args_info(self, setupscript='./setup.py'):
457 cmd = ['python3', setupscript]
458 info = {}
459 keys = set(self.bbvar_map.keys())
460 keys |= set(self.setuparg_list_fields)
461 keys |= set(self.setuparg_multi_line_values)
462 grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
463 for index, keys in grouped_keys:
464 if index == (True, False):
465 # Splitlines output for each arg as a list value
466 for key in keys:
467 arg = self.setuparg_map.get(key, key.lower())
468 try:
469 arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
470 except (OSError, subprocess.CalledProcessError):
471 pass
472 else:
473 info[key] = [l.rstrip() for l in arg_info.splitlines()]
474 elif index == (False, True):
475 # Entire output for each arg
476 for key in keys:
477 arg = self.setuparg_map.get(key, key.lower())
478 try:
479 arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
480 except (OSError, subprocess.CalledProcessError):
481 pass
482 else:
483 info[key] = arg_info
484 else:
485 info.update(self.get_setup_byline(list(keys), setupscript))
486 return info
487
488 def get_setup_byline(self, fields, setupscript='./setup.py'):
489 info = {}
490
491 cmd = ['python3', setupscript]
492 cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields)
493 try:
494 info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
495 except (OSError, subprocess.CalledProcessError):
496 pass
497 else:
498 if len(fields) != len(info_lines):
499 logger.error('Mismatch between setup.py output lines and number of fields')
500 sys.exit(1)
501
502 for lineno, line in enumerate(info_lines):
503 line = line.rstrip()
504 info[fields[lineno]] = line
505 return info
506
507 def get_pkginfo(self, pkginfo_fn):
508 msg = email.message_from_file(open(pkginfo_fn, 'r'))
509 msginfo = {}
510 for field in msg.keys():
511 values = msg.get_all(field)
512 if len(values) == 1:
513 msginfo[field] = values[0]
514 else:
515 msginfo[field] = values
516 return msginfo
517
518 def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
519 if 'Package-dir' in setup_info:
520 package_dir = setup_info['Package-dir']
521 else:
522 package_dir = {}
523
524 dist = setuptools.Distribution()
525
526 class PackageDir(setuptools.command.build_py.build_py):
527 def __init__(self, package_dir):
528 self.package_dir = package_dir
529 self.dist = dist
530 super().__init__(self.dist)
531
532 pd = PackageDir(package_dir)
533 to_scan = []
534 if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
535 if 'Py-modules' in setup_info:
536 for module in setup_info['Py-modules']:
537 try:
538 package, module = module.rsplit('.', 1)
539 except ValueError:
540 package, module = '.', module
541 module_path = os.path.join(pd.get_package_dir(package), module + '.py')
542 to_scan.append(module_path)
543
544 if 'Packages' in setup_info:
545 for package in setup_info['Packages']:
546 to_scan.append(pd.get_package_dir(package))
547
548 if 'Scripts' in setup_info:
549 to_scan.extend(setup_info['Scripts'])
550 else:
551 logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
552
553 if not to_scan:
554 to_scan = ['.']
555
556 logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
557
558 provided_packages = self.parse_pkgdata_for_python_packages()
559 scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
560 mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
561 for dep in scanned_deps:
562 mapped = provided_packages.get(dep)
563 if mapped:
564 logger.debug('Mapped %s to %s' % (dep, mapped))
565 mapped_deps.add(mapped)
566 else:
567 logger.debug('Could not map %s' % dep)
568 unmapped_deps.add(dep)
569 return mapped_deps, unmapped_deps
570
153 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): 571 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
572
154 if 'buildsystem' in handled: 573 if 'buildsystem' in handled:
155 return False 574 return False
156 575
576 logger.debug("Trying setup.py parser")
577
157 # Check for non-zero size setup.py files 578 # Check for non-zero size setup.py files
158 setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py']) 579 setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py'])
159 for fn in setupfiles: 580 for fn in setupfiles:
160 if os.path.getsize(fn): 581 if os.path.getsize(fn):
161 break 582 break
162 else: 583 else:
584 logger.debug("No setup.py found")
163 return False 585 return False
164 586
165 # setup.py is always parsed to get at certain required information, such as 587 # setup.py is always parsed to get at certain required information, such as
@@ -193,6 +615,18 @@ class PythonRecipeHandler(RecipeHandler):
193 continue 615 continue
194 616
195 if line.startswith('['): 617 if line.startswith('['):
618 # PACKAGECONFIG must not contain expressions or whitespace
619 line = line.replace(" ", "")
620 line = line.replace(':', "")
621 line = line.replace('.', "-dot-")
622 line = line.replace('"', "")
623 line = line.replace('<', "-smaller-")
624 line = line.replace('>', "-bigger-")
625 line = line.replace('_', "-")
626 line = line.replace('(', "")
627 line = line.replace(')', "")
628 line = line.replace('!', "-not-")
629 line = line.replace('=', "-equals-")
196 current_feature = line[1:-1] 630 current_feature = line[1:-1]
197 elif current_feature: 631 elif current_feature:
198 extras_req[current_feature].append(line) 632 extras_req[current_feature].append(line)
@@ -226,51 +660,16 @@ class PythonRecipeHandler(RecipeHandler):
226 660
227 if license_str: 661 if license_str:
228 for i, line in enumerate(lines_before): 662 for i, line in enumerate(lines_before):
229 if line.startswith('LICENSE = '): 663 if line.startswith('##LICENSE_PLACEHOLDER##'):
230 lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str) 664 lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str)
231 break 665 break
232 666
233 if 'Classifier' in info: 667 if 'Classifier' in info:
234 existing_licenses = info.get('License', '') 668 license = self.handle_classifier_license(info['Classifier'], info.get('License', ''))
235 licenses = [] 669 if license:
236 for classifier in info['Classifier']: 670 info['License'] = license
237 if classifier in self.classifier_license_map:
238 license = self.classifier_license_map[classifier]
239 if license == 'Apache' and 'Apache-2.0' in existing_licenses:
240 license = 'Apache-2.0'
241 elif license == 'GPL':
242 if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
243 license = 'GPL-2.0'
244 elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
245 license = 'GPL-3.0'
246 elif license == 'LGPL':
247 if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
248 license = 'LGPL-2.1'
249 elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
250 license = 'LGPL-2.0'
251 elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
252 license = 'LGPL-3.0'
253 licenses.append(license)
254
255 if licenses:
256 info['License'] = ' & '.join(licenses)
257 671
258 # Map PKG-INFO & setup.py fields to bitbake variables 672 self.map_info_to_bbvar(info, extravalues)
259 for field, values in info.items():
260 if field in self.excluded_fields:
261 continue
262
263 if field not in self.bbvar_map:
264 continue
265
266 if isinstance(values, str):
267 value = values
268 else:
269 value = ' '.join(str(v) for v in values if v)
270
271 bbvar = self.bbvar_map[field]
272 if bbvar not in extravalues and value:
273 extravalues[bbvar] = value
274 673
275 mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) 674 mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
276 675
@@ -281,6 +680,7 @@ class PythonRecipeHandler(RecipeHandler):
281 lines_after.append('# The following configs & dependencies are from setuptools extras_require.') 680 lines_after.append('# The following configs & dependencies are from setuptools extras_require.')
282 lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.') 681 lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.')
283 lines_after.append('# The upstream names may not correspond exactly to bitbake package names.') 682 lines_after.append('# The upstream names may not correspond exactly to bitbake package names.')
683 lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.')
284 lines_after.append('#') 684 lines_after.append('#')
285 lines_after.append('# Uncomment this line to enable all the optional features.') 685 lines_after.append('# Uncomment this line to enable all the optional features.')
286 lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req))) 686 lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req)))
@@ -301,7 +701,7 @@ class PythonRecipeHandler(RecipeHandler):
301 inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) 701 inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs))
302 lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') 702 lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These')
303 lines_after.append('# upstream names may not correspond exactly to bitbake package names.') 703 lines_after.append('# upstream names may not correspond exactly to bitbake package names.')
304 lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps))) 704 lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
305 705
306 if mapped_deps: 706 if mapped_deps:
307 name = info.get('Name') 707 name = info.get('Name')
@@ -313,7 +713,7 @@ class PythonRecipeHandler(RecipeHandler):
313 lines_after.append('') 713 lines_after.append('')
314 lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') 714 lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the')
315 lines_after.append('# python sources, and might not be 100% accurate.') 715 lines_after.append('# python sources, and might not be 100% accurate.')
316 lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) 716 lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
317 717
318 unmapped_deps -= set(extensions) 718 unmapped_deps -= set(extensions)
319 unmapped_deps -= set(self.assume_provided) 719 unmapped_deps -= set(self.assume_provided)
@@ -326,275 +726,283 @@ class PythonRecipeHandler(RecipeHandler):
326 726
327 handled.append('buildsystem') 727 handled.append('buildsystem')
328 728
329 def get_pkginfo(self, pkginfo_fn): 729class PythonPyprojectTomlRecipeHandler(PythonRecipeHandler):
330 msg = email.message_from_file(open(pkginfo_fn, 'r')) 730 """Base class to support PEP517 and PEP518
331 msginfo = {} 731
332 for field in msg.keys(): 732 PEP517 https://peps.python.org/pep-0517/#source-trees
333 values = msg.get_all(field) 733 PEP518 https://peps.python.org/pep-0518/#build-system-table
334 if len(values) == 1: 734 """
335 msginfo[field] = values[0] 735 # bitbake currently supports the 4 following backends
336 else: 736 build_backend_map = {
337 msginfo[field] = values 737 "setuptools.build_meta": "python_setuptools_build_meta",
338 return msginfo 738 "poetry.core.masonry.api": "python_poetry_core",
739 "flit_core.buildapi": "python_flit_core",
740 "hatchling.build": "python_hatchling",
741 "maturin": "python_maturin",
742 "mesonpy": "python_mesonpy",
743 }
339 744
340 def parse_setup_py(self, setupscript='./setup.py'): 745 # setuptools.build_meta and flit declare project metadata into the "project" section of pyproject.toml
341 with codecs.open(setupscript) as f: 746 # according to PEP-621: https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#declaring-project-metadata
342 info, imported_modules, non_literals, extensions = gather_setup_info(f) 747 # while poetry uses the "tool.poetry" section according to its official documentation: https://python-poetry.org/docs/pyproject/
748 # keys from "project" and "tool.poetry" sections are almost the same except for the HOMEPAGE which is "homepage" for tool.poetry
749 # and "Homepage" for "project" section. So keep both
750 bbvar_map = {
751 "name": "PN",
752 "version": "PV",
753 "Homepage": "HOMEPAGE",
754 "homepage": "HOMEPAGE",
755 "description": "SUMMARY",
756 "license": "LICENSE",
757 "dependencies": "RDEPENDS:${PN}",
758 "requires": "DEPENDS",
759 }
343 760
344 def _map(key): 761 replacements = [
345 key = key.replace('_', '-') 762 ("license", r" +$", ""),
346 key = key[0].upper() + key[1:] 763 ("license", r"^ +", ""),
347 if key in self.setup_parse_map: 764 ("license", r" ", "-"),
348 key = self.setup_parse_map[key] 765 ("license", r"^GNU-", ""),
349 return key 766 ("license", r"-[Ll]icen[cs]e(,?-[Vv]ersion)?", ""),
767 ("license", r"^UNKNOWN$", ""),
768 # Remove currently unhandled version numbers from these variables
769 ("requires", r"\[[^\]]+\]$", ""),
770 ("requires", r"^([^><= ]+).*", r"\1"),
771 ("dependencies", r"\[[^\]]+\]$", ""),
772 ("dependencies", r"^([^><= ]+).*", r"\1"),
773 ]
350 774
351 # Naive mapping of setup() arguments to PKG-INFO field names 775 excluded_native_pkgdeps = [
352 for d in [info, non_literals]: 776 # already provided by python_setuptools_build_meta.bbclass
353 for key, value in list(d.items()): 777 "python3-setuptools-native",
354 if key is None: 778 "python3-wheel-native",
355 continue 779 # already provided by python_poetry_core.bbclass
356 new_key = _map(key) 780 "python3-poetry-core-native",
357 if new_key != key: 781 # already provided by python_flit_core.bbclass
358 del d[key] 782 "python3-flit-core-native",
359 d[new_key] = value 783 # already provided by python_mesonpy
784 "python3-meson-python-native",
785 ]
360 786
361 return info, 'setuptools' in imported_modules, non_literals, extensions 787 # add here a list of known and often used packages and the corresponding bitbake package
788 known_deps_map = {
789 "setuptools": "python3-setuptools",
790 "wheel": "python3-wheel",
791 "poetry-core": "python3-poetry-core",
792 "flit_core": "python3-flit-core",
793 "setuptools-scm": "python3-setuptools-scm",
794 "hatchling": "python3-hatchling",
795 "hatch-vcs": "python3-hatch-vcs",
796 "meson-python" : "python3-meson-python",
797 }
362 798
363 def get_setup_args_info(self, setupscript='./setup.py'): 799 def __init__(self):
364 cmd = ['python3', setupscript] 800 pass
365 info = {}
366 keys = set(self.bbvar_map.keys())
367 keys |= set(self.setuparg_list_fields)
368 keys |= set(self.setuparg_multi_line_values)
369 grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
370 for index, keys in grouped_keys:
371 if index == (True, False):
372 # Splitlines output for each arg as a list value
373 for key in keys:
374 arg = self.setuparg_map.get(key, key.lower())
375 try:
376 arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
377 except (OSError, subprocess.CalledProcessError):
378 pass
379 else:
380 info[key] = [l.rstrip() for l in arg_info.splitlines()]
381 elif index == (False, True):
382 # Entire output for each arg
383 for key in keys:
384 arg = self.setuparg_map.get(key, key.lower())
385 try:
386 arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
387 except (OSError, subprocess.CalledProcessError):
388 pass
389 else:
390 info[key] = arg_info
391 else:
392 info.update(self.get_setup_byline(list(keys), setupscript))
393 return info
394 801
395 def get_setup_byline(self, fields, setupscript='./setup.py'): 802 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
396 info = {} 803 info = {}
804 metadata = {}
397 805
398 cmd = ['python3', setupscript] 806 if 'buildsystem' in handled:
399 cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields) 807 return False
400 try:
401 info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
402 except (OSError, subprocess.CalledProcessError):
403 pass
404 else:
405 if len(fields) != len(info_lines):
406 logger.error('Mismatch between setup.py output lines and number of fields')
407 sys.exit(1)
408
409 for lineno, line in enumerate(info_lines):
410 line = line.rstrip()
411 info[fields[lineno]] = line
412 return info
413
414 def apply_info_replacements(self, info):
415 for variable, search, replace in self.replacements:
416 if variable not in info:
417 continue
418
419 def replace_value(search, replace, value):
420 if replace is None:
421 if re.search(search, value):
422 return None
423 else:
424 new_value = re.sub(search, replace, value)
425 if value != new_value:
426 return new_value
427 return value
428
429 value = info[variable]
430 if isinstance(value, str):
431 new_value = replace_value(search, replace, value)
432 if new_value is None:
433 del info[variable]
434 elif new_value != value:
435 info[variable] = new_value
436 elif hasattr(value, 'items'):
437 for dkey, dvalue in list(value.items()):
438 new_list = []
439 for pos, a_value in enumerate(dvalue):
440 new_value = replace_value(search, replace, a_value)
441 if new_value is not None and new_value != value:
442 new_list.append(new_value)
443
444 if value != new_list:
445 value[dkey] = new_list
446 else:
447 new_list = []
448 for pos, a_value in enumerate(value):
449 new_value = replace_value(search, replace, a_value)
450 if new_value is not None and new_value != value:
451 new_list.append(new_value)
452
453 if value != new_list:
454 info[variable] = new_list
455
456 def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
457 if 'Package-dir' in setup_info:
458 package_dir = setup_info['Package-dir']
459 else:
460 package_dir = {}
461
462 class PackageDir(distutils.command.build_py.build_py):
463 def __init__(self, package_dir):
464 self.package_dir = package_dir
465
466 pd = PackageDir(package_dir)
467 to_scan = []
468 if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
469 if 'Py-modules' in setup_info:
470 for module in setup_info['Py-modules']:
471 try:
472 package, module = module.rsplit('.', 1)
473 except ValueError:
474 package, module = '.', module
475 module_path = os.path.join(pd.get_package_dir(package), module + '.py')
476 to_scan.append(module_path)
477 808
478 if 'Packages' in setup_info: 809 logger.debug("Trying pyproject.toml parser")
479 for package in setup_info['Packages']:
480 to_scan.append(pd.get_package_dir(package))
481 810
482 if 'Scripts' in setup_info: 811 # Check for non-zero size setup.py files
483 to_scan.extend(setup_info['Scripts']) 812 setupfiles = RecipeHandler.checkfiles(srctree, ["pyproject.toml"])
813 for fn in setupfiles:
814 if os.path.getsize(fn):
815 break
484 else: 816 else:
485 logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.") 817 logger.debug("No pyproject.toml found")
486 818 return False
487 if not to_scan:
488 to_scan = ['.']
489
490 logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
491 819
492 provided_packages = self.parse_pkgdata_for_python_packages() 820 setupscript = os.path.join(srctree, "pyproject.toml")
493 scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
494 mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
495 for dep in scanned_deps:
496 mapped = provided_packages.get(dep)
497 if mapped:
498 logger.debug('Mapped %s to %s' % (dep, mapped))
499 mapped_deps.add(mapped)
500 else:
501 logger.debug('Could not map %s' % dep)
502 unmapped_deps.add(dep)
503 return mapped_deps, unmapped_deps
504 821
505 def scan_python_dependencies(self, paths):
506 deps = set()
507 try: 822 try:
508 dep_output = self.run_command(['pythondeps', '-d'] + paths) 823 try:
509 except (OSError, subprocess.CalledProcessError): 824 import tomllib
510 pass 825 except ImportError:
511 else: 826 try:
512 for line in dep_output.splitlines(): 827 import tomli as tomllib
513 line = line.rstrip() 828 except ImportError:
514 dep, filename = line.split('\t', 1) 829 logger.error("Neither 'tomllib' nor 'tomli' could be imported, cannot scan pyproject.toml.")
515 if filename.endswith('/setup.py'): 830 return False
516 continue 831
517 deps.add(dep) 832 try:
833 with open(setupscript, "rb") as f:
834 config = tomllib.load(f)
835 except Exception:
836 logger.exception("Failed to parse pyproject.toml")
837 return False
838
839 build_backend = config["build-system"]["build-backend"]
840 if build_backend in self.build_backend_map:
841 classes.append(self.build_backend_map[build_backend])
842 else:
843 logger.error(
844 "Unsupported build-backend: %s, cannot use pyproject.toml. Will try to use legacy setup.py"
845 % build_backend
846 )
847 return False
518 848
519 try: 849 licfile = ""
520 provides_output = self.run_command(['pythondeps', '-p'] + paths)
521 except (OSError, subprocess.CalledProcessError):
522 pass
523 else:
524 provides_lines = (l.rstrip() for l in provides_output.splitlines())
525 provides = set(l for l in provides_lines if l and l != 'setup')
526 deps -= provides
527 850
528 return deps 851 if build_backend == "poetry.core.masonry.api":
852 if "tool" in config and "poetry" in config["tool"]:
853 metadata = config["tool"]["poetry"]
854 else:
855 if "project" in config:
856 metadata = config["project"]
857
858 if metadata:
859 for field, values in metadata.items():
860 if field == "license":
861 # For setuptools.build_meta and flit, licence is a table
862 # but for poetry licence is a string
863 # for hatchling, both table (jsonschema) and string (iniconfig) have been used
864 if build_backend == "poetry.core.masonry.api":
865 value = values
866 else:
867 value = values.get("text", "")
868 if not value:
869 licfile = values.get("file", "")
870 continue
871 elif field == "dependencies" and build_backend == "poetry.core.masonry.api":
872 # For poetry backend, "dependencies" section looks like:
873 # [tool.poetry.dependencies]
874 # requests = "^2.13.0"
875 # requests = { version = "^2.13.0", source = "private" }
876 # See https://python-poetry.org/docs/master/pyproject/#dependencies-and-dependency-groups for more details
877 # This class doesn't handle versions anyway, so we just get the dependencies name here and construct a list
878 value = []
879 for k in values.keys():
880 value.append(k)
881 elif isinstance(values, dict):
882 for k, v in values.items():
883 info[k] = v
884 continue
885 else:
886 value = values
529 887
530 def parse_pkgdata_for_python_packages(self): 888 info[field] = value
531 suffixes = [t[0] for t in imp.get_suffixes()]
532 pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
533 889
534 ldata = tinfoil.config_data.createCopy() 890 # Grab the license value before applying replacements
535 bb.parse.handle('classes/python3-dir.bbclass', ldata, True) 891 license_str = info.get("license", "").strip()
536 python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
537 892
538 dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') 893 if license_str:
539 python_dirs = [python_sitedir + os.sep, 894 for i, line in enumerate(lines_before):
540 os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, 895 if line.startswith("##LICENSE_PLACEHOLDER##"):
541 os.path.dirname(python_sitedir) + os.sep] 896 lines_before.insert(
542 packages = {} 897 i, "# NOTE: License in pyproject.toml is: %s" % license_str
543 for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): 898 )
544 files_info = None
545 with open(pkgdatafile, 'r') as f:
546 for line in f.readlines():
547 field, value = line.split(': ', 1)
548 if field == 'FILES_INFO':
549 files_info = ast.literal_eval(value)
550 break 899 break
551 else:
552 continue
553 900
554 for fn in files_info: 901 info["requires"] = config["build-system"]["requires"]
555 for suffix in suffixes: 902
556 if fn.endswith(suffix): 903 self.apply_info_replacements(info)
557 break 904
558 else: 905 if "classifiers" in info:
559 continue 906 license = self.handle_classifier_license(
907 info["classifiers"], info.get("license", "")
908 )
909 if license:
910 if licfile:
911 lines = []
912 md5value = bb.utils.md5_file(os.path.join(srctree, licfile))
913 lines.append('LICENSE = "%s"' % license)
914 lines.append(
915 'LIC_FILES_CHKSUM = "file://%s;md5=%s"'
916 % (licfile, md5value)
917 )
918 lines.append("")
919
920 # Replace the placeholder so we get the values in the right place in the recipe file
921 try:
922 pos = lines_before.index("##LICENSE_PLACEHOLDER##")
923 except ValueError:
924 pos = -1
925 if pos == -1:
926 lines_before.extend(lines)
927 else:
928 lines_before[pos : pos + 1] = lines
560 929
561 if fn.startswith(dynload_dir + os.sep): 930 handled.append(("license", [license, licfile, md5value]))
562 if '/.debug/' in fn: 931 else:
563 continue 932 info["license"] = license
564 base = os.path.basename(fn)
565 provided = base.split('.', 1)[0]
566 packages[provided] = os.path.basename(pkgdatafile)
567 continue
568 933
569 for python_dir in python_dirs: 934 provided_packages = self.parse_pkgdata_for_python_packages()
570 if fn.startswith(python_dir): 935 provided_packages.update(self.known_deps_map)
571 relpath = fn[len(python_dir):] 936 native_mapped_deps, native_unmapped_deps = set(), set()
572 relstart, _, relremaining = relpath.partition(os.sep) 937 mapped_deps, unmapped_deps = set(), set()
573 if relstart.endswith('.egg'):
574 relpath = relremaining
575 base, _ = os.path.splitext(relpath)
576 938
577 if '/.debug/' in base: 939 if "requires" in info:
578 continue 940 for require in info["requires"]:
579 if os.path.basename(base) == '__init__': 941 mapped = provided_packages.get(require)
580 base = os.path.dirname(base)
581 base = base.replace(os.sep + os.sep, os.sep)
582 provided = base.replace(os.sep, '.')
583 packages[provided] = os.path.basename(pkgdatafile)
584 return packages
585 942
586 @classmethod 943 if mapped:
587 def run_command(cls, cmd, **popenargs): 944 logger.debug("Mapped %s to %s" % (require, mapped))
588 if 'stderr' not in popenargs: 945 native_mapped_deps.add(mapped)
589 popenargs['stderr'] = subprocess.STDOUT 946 else:
590 try: 947 logger.debug("Could not map %s" % require)
591 return subprocess.check_output(cmd, **popenargs).decode('utf-8') 948 native_unmapped_deps.add(require)
592 except OSError as exc: 949
593 logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc) 950 info.pop("requires")
594 raise 951
595 except subprocess.CalledProcessError as exc: 952 if native_mapped_deps != set():
596 logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output) 953 native_mapped_deps = {
597 raise 954 item + "-native" for item in native_mapped_deps
955 }
956 native_mapped_deps -= set(self.excluded_native_pkgdeps)
957 if native_mapped_deps != set():
958 info["requires"] = " ".join(sorted(native_mapped_deps))
959
960 if native_unmapped_deps:
961 lines_after.append("")
962 lines_after.append(
963 "# WARNING: We were unable to map the following python package/module"
964 )
965 lines_after.append(
966 "# dependencies to the bitbake packages which include them:"
967 )
968 lines_after.extend(
969 "# {}".format(d) for d in sorted(native_unmapped_deps)
970 )
971
972 if "dependencies" in info:
973 for dependency in info["dependencies"]:
974 mapped = provided_packages.get(dependency)
975 if mapped:
976 logger.debug("Mapped %s to %s" % (dependency, mapped))
977 mapped_deps.add(mapped)
978 else:
979 logger.debug("Could not map %s" % dependency)
980 unmapped_deps.add(dependency)
981
982 info.pop("dependencies")
983
984 if mapped_deps != set():
985 if mapped_deps != set():
986 info["dependencies"] = " ".join(sorted(mapped_deps))
987
988 if unmapped_deps:
989 lines_after.append("")
990 lines_after.append(
991 "# WARNING: We were unable to map the following python package/module"
992 )
993 lines_after.append(
994 "# runtime dependencies to the bitbake packages which include them:"
995 )
996 lines_after.extend(
997 "# {}".format(d) for d in sorted(unmapped_deps)
998 )
999
1000 self.map_info_to_bbvar(info, extravalues)
1001
1002 handled.append("buildsystem")
1003 except Exception:
1004 logger.exception("Failed to correctly handle pyproject.toml, falling back to another method")
1005 return False
598 1006
599 1007
600def gather_setup_info(fileobj): 1008def gather_setup_info(fileobj):
@@ -710,5 +1118,7 @@ def has_non_literals(value):
710 1118
711 1119
712def register_recipe_handlers(handlers): 1120def register_recipe_handlers(handlers):
713 # We need to make sure this is ahead of the makefile fallback handler 1121 # We need to make sure these are ahead of the makefile fallback handler
714 handlers.append((PythonRecipeHandler(), 70)) 1122 # and the pyproject.toml handler ahead of the setup.py handler
1123 handlers.append((PythonPyprojectTomlRecipeHandler(), 75))
1124 handlers.append((PythonSetupPyRecipeHandler(), 70))
diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py
new file mode 100644
index 0000000000..a85a2f2786
--- /dev/null
+++ b/scripts/lib/recipetool/create_go.py
@@ -0,0 +1,777 @@
1# Recipe creation tool - go support plugin
2#
3# The code is based on golang internals. See the afftected
4# methods for further reference and information.
5#
6# Copyright (C) 2023 Weidmueller GmbH & Co KG
7# Author: Lukas Funke <lukas.funke@weidmueller.com>
8#
9# SPDX-License-Identifier: GPL-2.0-only
10#
11
12
13from collections import namedtuple
14from enum import Enum
15from html.parser import HTMLParser
16from recipetool.create import RecipeHandler, handle_license_vars
17from recipetool.create import guess_license, tidy_licenses, fixup_license
18from recipetool.create import determine_from_url
19from urllib.error import URLError, HTTPError
20
21import bb.utils
22import json
23import logging
24import os
25import re
26import subprocess
27import sys
28import shutil
29import tempfile
30import urllib.parse
31import urllib.request
32
33
34GoImport = namedtuple('GoImport', 'root vcs url suffix')
35logger = logging.getLogger('recipetool')
36CodeRepo = namedtuple(
37 'CodeRepo', 'path codeRoot codeDir pathMajor pathPrefix pseudoMajor')
38
39tinfoil = None
40
41# Regular expression to parse pseudo semantic version
42# see https://go.dev/ref/mod#pseudo-versions
43re_pseudo_semver = re.compile(
44 r"^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)(?P<utc>\d{14})-(?P<commithash>[A-Za-z0-9]+)(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$")
45# Regular expression to parse semantic version
46re_semver = re.compile(
47 r"^v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$")
48
49
50def tinfoil_init(instance):
51 global tinfoil
52 tinfoil = instance
53
54
55class GoRecipeHandler(RecipeHandler):
56 """Class to handle the go recipe creation"""
57
58 @staticmethod
59 def __ensure_go():
60 """Check if the 'go' command is available in the recipes"""
61 recipe = "go-native"
62 if not tinfoil.recipes_parsed:
63 tinfoil.parse_recipes()
64 try:
65 rd = tinfoil.parse_recipe(recipe)
66 except bb.providers.NoProvider:
67 bb.error(
68 "Nothing provides '%s' which is required for the build" % (recipe))
69 bb.note(
70 "You will likely need to add a layer that provides '%s'" % (recipe))
71 return None
72
73 bindir = rd.getVar('STAGING_BINDIR_NATIVE')
74 gopath = os.path.join(bindir, 'go')
75
76 if not os.path.exists(gopath):
77 tinfoil.build_targets(recipe, 'addto_recipe_sysroot')
78
79 if not os.path.exists(gopath):
80 logger.error(
81 '%s required to process specified source, but %s did not seem to populate it' % 'go', recipe)
82 return None
83
84 return bindir
85
86 def __resolve_repository_static(self, modulepath):
87 """Resolve the repository in a static manner
88
89 The method is based on the go implementation of
90 `repoRootFromVCSPaths` in
91 https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go
92 """
93
94 url = urllib.parse.urlparse("https://" + modulepath)
95 req = urllib.request.Request(url.geturl())
96
97 try:
98 resp = urllib.request.urlopen(req)
99 # Some modulepath are just redirects to github (or some other vcs
100 # hoster). Therefore, we check if this modulepath redirects to
101 # somewhere else
102 if resp.geturl() != url.geturl():
103 bb.debug(1, "%s is redirectred to %s" %
104 (url.geturl(), resp.geturl()))
105 url = urllib.parse.urlparse(resp.geturl())
106 modulepath = url.netloc + url.path
107
108 except URLError as url_err:
109 # This is probably because the module path
110 # contains the subdir and major path. Thus,
111 # we ignore this error for now
112 logger.debug(
113 1, "Failed to fetch page from [%s]: %s" % (url, str(url_err)))
114
115 host, _, _ = modulepath.partition('/')
116
117 class vcs(Enum):
118 pathprefix = "pathprefix"
119 regexp = "regexp"
120 type = "type"
121 repo = "repo"
122 check = "check"
123 schemelessRepo = "schemelessRepo"
124
125 # GitHub
126 vcsGitHub = {}
127 vcsGitHub[vcs.pathprefix] = "github.com"
128 vcsGitHub[vcs.regexp] = re.compile(
129 r'^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
130 vcsGitHub[vcs.type] = "git"
131 vcsGitHub[vcs.repo] = "https://\\g<root>"
132
133 # Bitbucket
134 vcsBitbucket = {}
135 vcsBitbucket[vcs.pathprefix] = "bitbucket.org"
136 vcsBitbucket[vcs.regexp] = re.compile(
137 r'^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
138 vcsBitbucket[vcs.type] = "git"
139 vcsBitbucket[vcs.repo] = "https://\\g<root>"
140
141 # IBM DevOps Services (JazzHub)
142 vcsIBMDevOps = {}
143 vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git"
144 vcsIBMDevOps[vcs.regexp] = re.compile(
145 r'^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
146 vcsIBMDevOps[vcs.type] = "git"
147 vcsIBMDevOps[vcs.repo] = "https://\\g<root>"
148
149 # Git at Apache
150 vcsApacheGit = {}
151 vcsApacheGit[vcs.pathprefix] = "git.apache.org"
152 vcsApacheGit[vcs.regexp] = re.compile(
153 r'^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
154 vcsApacheGit[vcs.type] = "git"
155 vcsApacheGit[vcs.repo] = "https://\\g<root>"
156
157 # Git at OpenStack
158 vcsOpenStackGit = {}
159 vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org"
160 vcsOpenStackGit[vcs.regexp] = re.compile(
161 r'^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
162 vcsOpenStackGit[vcs.type] = "git"
163 vcsOpenStackGit[vcs.repo] = "https://\\g<root>"
164
165 # chiselapp.com for fossil
166 vcsChiselapp = {}
167 vcsChiselapp[vcs.pathprefix] = "chiselapp.com"
168 vcsChiselapp[vcs.regexp] = re.compile(
169 r'^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$')
170 vcsChiselapp[vcs.type] = "fossil"
171 vcsChiselapp[vcs.repo] = "https://\\g<root>"
172
173 # General syntax for any server.
174 # Must be last.
175 vcsGeneralServer = {}
176 vcsGeneralServer[vcs.regexp] = re.compile(
177 "(?P<root>(?P<repo>([a-z0-9.\\-]+\\.)+[a-z0-9.\\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\\-]+)+?)\\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?(?P<suffix>[A-Za-z0-9_.\\-]+))*$")
178 vcsGeneralServer[vcs.schemelessRepo] = True
179
180 vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps,
181 vcsApacheGit, vcsOpenStackGit, vcsChiselapp,
182 vcsGeneralServer]
183
184 if modulepath.startswith("example.net") or modulepath == "rsc.io":
185 logger.warning("Suspicious module path %s" % modulepath)
186 return None
187 if modulepath.startswith("http:") or modulepath.startswith("https:"):
188 logger.warning("Import path should not start with %s %s" %
189 ("http", "https"))
190 return None
191
192 rootpath = None
193 vcstype = None
194 repourl = None
195 suffix = None
196
197 for srv in vcsPaths:
198 m = srv[vcs.regexp].match(modulepath)
199 if vcs.pathprefix in srv:
200 if host == srv[vcs.pathprefix]:
201 rootpath = m.group('root')
202 vcstype = srv[vcs.type]
203 repourl = m.expand(srv[vcs.repo])
204 suffix = m.group('suffix')
205 break
206 elif m and srv[vcs.schemelessRepo]:
207 rootpath = m.group('root')
208 vcstype = m[vcs.type]
209 repourl = m[vcs.repo]
210 suffix = m.group('suffix')
211 break
212
213 return GoImport(rootpath, vcstype, repourl, suffix)
214
215 def __resolve_repository_dynamic(self, modulepath):
216 """Resolve the repository root in a dynamic manner.
217
218 The method is based on the go implementation of
219 `repoRootForImportDynamic` in
220 https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go
221 """
222 url = urllib.parse.urlparse("https://" + modulepath)
223
224 class GoImportHTMLParser(HTMLParser):
225
226 def __init__(self):
227 super().__init__()
228 self.__srv = {}
229
230 def handle_starttag(self, tag, attrs):
231 if tag == 'meta' and list(
232 filter(lambda a: (a[0] == 'name' and a[1] == 'go-import'), attrs)):
233 content = list(
234 filter(lambda a: (a[0] == 'content'), attrs))
235 if content:
236 srv = content[0][1].split()
237 self.__srv[srv[0]] = srv
238
239 def go_import(self, modulepath):
240 if modulepath in self.__srv:
241 srv = self.__srv[modulepath]
242 return GoImport(srv[0], srv[1], srv[2], None)
243 return None
244
245 url = url.geturl() + "?go-get=1"
246 req = urllib.request.Request(url)
247
248 try:
249 body = urllib.request.urlopen(req).read()
250 except HTTPError as http_err:
251 logger.warning(
252 "Unclean status when fetching page from [%s]: %s", url, str(http_err))
253 body = http_err.fp.read()
254 except URLError as url_err:
255 logger.warning(
256 "Failed to fetch page from [%s]: %s", url, str(url_err))
257 return None
258
259 parser = GoImportHTMLParser()
260 parser.feed(body.decode('utf-8'))
261 parser.close()
262
263 return parser.go_import(modulepath)
264
265 def __resolve_from_golang_proxy(self, modulepath, version):
266 """
267 Resolves repository data from golang proxy
268 """
269 url = urllib.parse.urlparse("https://proxy.golang.org/"
270 + modulepath
271 + "/@v/"
272 + version
273 + ".info")
274
275 # Transform url to lower case, golang proxy doesn't like mixed case
276 req = urllib.request.Request(url.geturl().lower())
277
278 try:
279 resp = urllib.request.urlopen(req)
280 except URLError as url_err:
281 logger.warning(
282 "Failed to fetch page from [%s]: %s", url, str(url_err))
283 return None
284
285 golang_proxy_res = resp.read().decode('utf-8')
286 modinfo = json.loads(golang_proxy_res)
287
288 if modinfo and 'Origin' in modinfo:
289 origin = modinfo['Origin']
290 _root_url = urllib.parse.urlparse(origin['URL'])
291
292 # We normalize the repo URL since we don't want the scheme in it
293 _subdir = origin['Subdir'] if 'Subdir' in origin else None
294 _root, _, _ = self.__split_path_version(modulepath)
295 if _subdir:
296 _root = _root[:-len(_subdir)].strip('/')
297
298 _commit = origin['Hash']
299 _vcs = origin['VCS']
300 return (GoImport(_root, _vcs, _root_url.geturl(), None), _commit)
301
302 return None
303
304 def __resolve_repository(self, modulepath):
305 """
306 Resolves src uri from go module-path
307 """
308 repodata = self.__resolve_repository_static(modulepath)
309 if not repodata or not repodata.url:
310 repodata = self.__resolve_repository_dynamic(modulepath)
311 if not repodata or not repodata.url:
312 logger.error(
313 "Could not resolve repository for module path '%s'" % modulepath)
314 # There is no way to recover from this
315 sys.exit(14)
316 if repodata:
317 logger.debug(1, "Resolved download path for import '%s' => %s" % (
318 modulepath, repodata.url))
319 return repodata
320
321 def __split_path_version(self, path):
322 i = len(path)
323 dot = False
324 for j in range(i, 0, -1):
325 if path[j - 1] < '0' or path[j - 1] > '9':
326 break
327 if path[j - 1] == '.':
328 dot = True
329 break
330 i = j - 1
331
332 if i <= 1 or i == len(
333 path) or path[i - 1] != 'v' or path[i - 2] != '/':
334 return path, "", True
335
336 prefix, pathMajor = path[:i - 2], path[i - 2:]
337 if dot or len(
338 pathMajor) <= 2 or pathMajor[2] == '0' or pathMajor == "/v1":
339 return path, "", False
340
341 return prefix, pathMajor, True
342
343 def __get_path_major(self, pathMajor):
344 if not pathMajor:
345 return ""
346
347 if pathMajor[0] != '/' and pathMajor[0] != '.':
348 logger.error(
349 "pathMajor suffix %s passed to PathMajorPrefix lacks separator", pathMajor)
350
351 if pathMajor.startswith(".v") and pathMajor.endswith("-unstable"):
352 pathMajor = pathMajor[:len("-unstable") - 2]
353
354 return pathMajor[1:]
355
356 def __build_coderepo(self, repo, path):
357 codedir = ""
358 pathprefix, pathMajor, _ = self.__split_path_version(path)
359 if repo.root == path:
360 pathprefix = path
361 elif path.startswith(repo.root):
362 codedir = pathprefix[len(repo.root):].strip('/')
363
364 pseudoMajor = self.__get_path_major(pathMajor)
365
366 logger.debug("root='%s', codedir='%s', prefix='%s', pathMajor='%s', pseudoMajor='%s'",
367 repo.root, codedir, pathprefix, pathMajor, pseudoMajor)
368
369 return CodeRepo(path, repo.root, codedir,
370 pathMajor, pathprefix, pseudoMajor)
371
372 def __resolve_version(self, repo, path, version):
373 hash = None
374 coderoot = self.__build_coderepo(repo, path)
375
376 def vcs_fetch_all():
377 tmpdir = tempfile.mkdtemp()
378 clone_cmd = "%s clone --bare %s %s" % ('git', repo.url, tmpdir)
379 bb.process.run(clone_cmd)
380 log_cmd = "git log --all --pretty='%H %d' --decorate=short"
381 output, _ = bb.process.run(
382 log_cmd, shell=True, stderr=subprocess.PIPE, cwd=tmpdir)
383 bb.utils.prunedir(tmpdir)
384 return output.strip().split('\n')
385
386 def vcs_fetch_remote(tag):
387 # add * to grab ^{}
388 refs = {}
389 ls_remote_cmd = "git ls-remote -q --tags {} {}*".format(
390 repo.url, tag)
391 output, _ = bb.process.run(ls_remote_cmd)
392 output = output.strip().split('\n')
393 for line in output:
394 f = line.split(maxsplit=1)
395 if len(f) != 2:
396 continue
397
398 for prefix in ["HEAD", "refs/heads/", "refs/tags/"]:
399 if f[1].startswith(prefix):
400 refs[f[1][len(prefix):]] = f[0]
401
402 for key, hash in refs.items():
403 if key.endswith(r"^{}"):
404 refs[key.strip(r"^{}")] = hash
405
406 return refs[tag]
407
408 m_pseudo_semver = re_pseudo_semver.match(version)
409
410 if m_pseudo_semver:
411 remote_refs = vcs_fetch_all()
412 short_commit = m_pseudo_semver.group('commithash')
413 for l in remote_refs:
414 r = l.split(maxsplit=1)
415 sha1 = r[0] if len(r) else None
416 if not sha1:
417 logger.error(
418 "Ups: could not resolve abbref commit for %s" % short_commit)
419
420 elif sha1.startswith(short_commit):
421 hash = sha1
422 break
423 else:
424 m_semver = re_semver.match(version)
425 if m_semver:
426
427 def get_sha1_remote(re):
428 rsha1 = None
429 for line in remote_refs:
430 # Split lines of the following format:
431 # 22e90d9b964610628c10f673ca5f85b8c2a2ca9a (tag: sometag)
432 lineparts = line.split(maxsplit=1)
433 sha1 = lineparts[0] if len(lineparts) else None
434 refstring = lineparts[1] if len(
435 lineparts) == 2 else None
436 if refstring:
437 # Normalize tag string and split in case of multiple
438 # regs e.g. (tag: speech/v1.10.0, tag: orchestration/v1.5.0 ...)
439 refs = refstring.strip('(), ').split(',')
440 for ref in refs:
441 if re.match(ref.strip()):
442 rsha1 = sha1
443 return rsha1
444
445 semver = "v" + m_semver.group('major') + "."\
446 + m_semver.group('minor') + "."\
447 + m_semver.group('patch') \
448 + (("-" + m_semver.group('prerelease'))
449 if m_semver.group('prerelease') else "")
450
451 tag = os.path.join(
452 coderoot.codeDir, semver) if coderoot.codeDir else semver
453
454 # probe tag using 'ls-remote', which is faster than fetching
455 # complete history
456 hash = vcs_fetch_remote(tag)
457 if not hash:
458 # backup: fetch complete history
459 remote_refs = vcs_fetch_all()
460 hash = get_sha1_remote(
461 re.compile(fr"(tag:|HEAD ->) ({tag})"))
462
463 logger.debug(
464 "Resolving commit for tag '%s' -> '%s'", tag, hash)
465 return hash
466
467 def __generate_srcuri_inline_fcn(self, path, version, replaces=None):
468 """Generate SRC_URI functions for go imports"""
469
470 logger.info("Resolving repository for module %s", path)
471 # First try to resolve repo and commit from golang proxy
472 # Most info is already there and we don't have to go through the
473 # repository or even perform the version resolve magic
474 golang_proxy_info = self.__resolve_from_golang_proxy(path, version)
475 if golang_proxy_info:
476 repo = golang_proxy_info[0]
477 commit = golang_proxy_info[1]
478 else:
479 # Fallback
480 # Resolve repository by 'hand'
481 repo = self.__resolve_repository(path)
482 commit = self.__resolve_version(repo, path, version)
483
484 url = urllib.parse.urlparse(repo.url)
485 repo_url = url.netloc + url.path
486
487 coderoot = self.__build_coderepo(repo, path)
488
489 inline_fcn = "${@go_src_uri("
490 inline_fcn += f"'{repo_url}','{version}'"
491 if repo_url != path:
492 inline_fcn += f",path='{path}'"
493 if coderoot.codeDir:
494 inline_fcn += f",subdir='{coderoot.codeDir}'"
495 if repo.vcs != 'git':
496 inline_fcn += f",vcs='{repo.vcs}'"
497 if replaces:
498 inline_fcn += f",replaces='{replaces}'"
499 if coderoot.pathMajor:
500 inline_fcn += f",pathmajor='{coderoot.pathMajor}'"
501 inline_fcn += ")}"
502
503 return inline_fcn, commit
504
505 def __go_handle_dependencies(self, go_mod, srctree, localfilesdir, extravalues, d):
506
507 import re
508 src_uris = []
509 src_revs = []
510
511 def generate_src_rev(path, version, commithash):
512 src_rev = f"# {path}@{version} => {commithash}\n"
513 # Ups...maybe someone manipulated the source repository and the
514 # version or commit could not be resolved. This is a sign of
515 # a) the supply chain was manipulated (bad)
516 # b) the implementation for the version resolving didn't work
517 # anymore (less bad)
518 if not commithash:
519 src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
520 src_rev += f"#!!! Could not resolve version !!!\n"
521 src_rev += f"#!!! Possible supply chain attack !!!\n"
522 src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
523 src_rev += f"SRCREV_{path.replace('/', '.')} = \"{commithash}\""
524
525 return src_rev
526
527 # we first go over replacement list, because we are essentialy
528 # interested only in the replaced path
529 if go_mod['Replace']:
530 for replacement in go_mod['Replace']:
531 oldpath = replacement['Old']['Path']
532 path = replacement['New']['Path']
533 version = ''
534 if 'Version' in replacement['New']:
535 version = replacement['New']['Version']
536
537 if os.path.exists(os.path.join(srctree, path)):
538 # the module refers to the local path, remove it from requirement list
539 # because it's a local module
540 go_mod['Require'][:] = [v for v in go_mod['Require'] if v.get('Path') != oldpath]
541 else:
542 # Replace the path and the version, so we don't iterate replacement list anymore
543 for require in go_mod['Require']:
544 if require['Path'] == oldpath:
545 require.update({'Path': path, 'Version': version})
546 break
547
548 for require in go_mod['Require']:
549 path = require['Path']
550 version = require['Version']
551
552 inline_fcn, commithash = self.__generate_srcuri_inline_fcn(
553 path, version)
554 src_uris.append(inline_fcn)
555 src_revs.append(generate_src_rev(path, version, commithash))
556
557 # strip version part from module URL /vXX
558 baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
559 pn, _ = determine_from_url(baseurl)
560 go_mods_basename = "%s-modules.inc" % pn
561
562 go_mods_filename = os.path.join(localfilesdir, go_mods_basename)
563 with open(go_mods_filename, "w") as f:
564 # We introduce this indirection to make the tests a little easier
565 f.write("SRC_URI += \"${GO_DEPENDENCIES_SRC_URI}\"\n")
566 f.write("GO_DEPENDENCIES_SRC_URI = \"\\\n")
567 for uri in src_uris:
568 f.write(" " + uri + " \\\n")
569 f.write("\"\n\n")
570 for rev in src_revs:
571 f.write(rev + "\n")
572
573 extravalues['extrafiles'][go_mods_basename] = go_mods_filename
574
575 def __go_run_cmd(self, cmd, cwd, d):
576 return bb.process.run(cmd, env=dict(os.environ, PATH=d.getVar('PATH')),
577 shell=True, cwd=cwd)
578
579 def __go_native_version(self, d):
580 stdout, _ = self.__go_run_cmd("go version", None, d)
581 m = re.match(r".*\sgo((\d+).(\d+).(\d+))\s([\w\/]*)", stdout)
582 major = int(m.group(2))
583 minor = int(m.group(3))
584 patch = int(m.group(4))
585
586 return major, minor, patch
587
588 def __go_mod_patch(self, srctree, localfilesdir, extravalues, d):
589
590 patchfilename = "go.mod.patch"
591 go_native_version_major, go_native_version_minor, _ = self.__go_native_version(
592 d)
593 self.__go_run_cmd("go mod tidy -go=%d.%d" %
594 (go_native_version_major, go_native_version_minor), srctree, d)
595 stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d)
596
597 # Create patch in order to upgrade go version
598 self.__go_run_cmd("git diff go.mod > %s" % (patchfilename), srctree, d)
599 # Restore original state
600 self.__go_run_cmd("git checkout HEAD go.mod go.sum", srctree, d)
601
602 go_mod = json.loads(stdout)
603 tmpfile = os.path.join(localfilesdir, patchfilename)
604 shutil.move(os.path.join(srctree, patchfilename), tmpfile)
605
606 extravalues['extrafiles'][patchfilename] = tmpfile
607
608 return go_mod, patchfilename
609
610 def __go_mod_vendor(self, go_mod, srctree, localfilesdir, extravalues, d):
611 # Perform vendoring to retrieve the correct modules.txt
612 tmp_vendor_dir = tempfile.mkdtemp()
613
614 # -v causes to go to print modules.txt to stderr
615 _, stderr = self.__go_run_cmd(
616 "go mod vendor -v -o %s" % (tmp_vendor_dir), srctree, d)
617
618 modules_txt_basename = "modules.txt"
619 modules_txt_filename = os.path.join(localfilesdir, modules_txt_basename)
620 with open(modules_txt_filename, "w") as f:
621 f.write(stderr)
622
623 extravalues['extrafiles'][modules_txt_basename] = modules_txt_filename
624
625 licenses = []
626 lic_files_chksum = []
627 licvalues = guess_license(tmp_vendor_dir, d)
628 shutil.rmtree(tmp_vendor_dir)
629
630 if licvalues:
631 for licvalue in licvalues:
632 license = licvalue[0]
633 lics = tidy_licenses(fixup_license(license))
634 lics = [lic for lic in lics if lic not in licenses]
635 if len(lics):
636 licenses.extend(lics)
637 lic_files_chksum.append(
638 'file://src/${GO_IMPORT}/vendor/%s;md5=%s' % (licvalue[1], licvalue[2]))
639
640 # strip version part from module URL /vXX
641 baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
642 pn, _ = determine_from_url(baseurl)
643 licenses_basename = "%s-licenses.inc" % pn
644
645 licenses_filename = os.path.join(localfilesdir, licenses_basename)
646 with open(licenses_filename, "w") as f:
647 f.write("GO_MOD_LICENSES = \"%s\"\n\n" %
648 ' & '.join(sorted(licenses, key=str.casefold)))
649 # We introduce this indirection to make the tests a little easier
650 f.write("LIC_FILES_CHKSUM += \"${VENDORED_LIC_FILES_CHKSUM}\"\n")
651 f.write("VENDORED_LIC_FILES_CHKSUM = \"\\\n")
652 for lic in lic_files_chksum:
653 f.write(" " + lic + " \\\n")
654 f.write("\"\n")
655
656 extravalues['extrafiles'][licenses_basename] = licenses_filename
657
658 def process(self, srctree, classes, lines_before,
659 lines_after, handled, extravalues):
660
661 if 'buildsystem' in handled:
662 return False
663
664 files = RecipeHandler.checkfiles(srctree, ['go.mod'])
665 if not files:
666 return False
667
668 d = bb.data.createCopy(tinfoil.config_data)
669 go_bindir = self.__ensure_go()
670 if not go_bindir:
671 sys.exit(14)
672
673 d.prependVar('PATH', '%s:' % go_bindir)
674 handled.append('buildsystem')
675 classes.append("go-vendor")
676
677 stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d)
678
679 go_mod = json.loads(stdout)
680 go_import = go_mod['Module']['Path']
681 go_version_match = re.match("([0-9]+).([0-9]+)", go_mod['Go'])
682 go_version_major = int(go_version_match.group(1))
683 go_version_minor = int(go_version_match.group(2))
684 src_uris = []
685
686 localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
687 extravalues.setdefault('extrafiles', {})
688
689 # Use an explicit name determined from the module name because it
690 # might differ from the actual URL for replaced modules
691 # strip version part from module URL /vXX
692 baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
693 pn, _ = determine_from_url(baseurl)
694
695 # go.mod files with version < 1.17 may not include all indirect
696 # dependencies. Thus, we have to upgrade the go version.
697 if go_version_major == 1 and go_version_minor < 17:
698 logger.warning(
699 "go.mod files generated by Go < 1.17 might have incomplete indirect dependencies.")
700 go_mod, patchfilename = self.__go_mod_patch(srctree, localfilesdir,
701 extravalues, d)
702 src_uris.append(
703 "file://%s;patchdir=src/${GO_IMPORT}" % (patchfilename))
704
705 # Check whether the module is vendored. If so, we have nothing to do.
706 # Otherwise we gather all dependencies and add them to the recipe
707 if not os.path.exists(os.path.join(srctree, "vendor")):
708
709 # Write additional $BPN-modules.inc file
710 self.__go_mod_vendor(go_mod, srctree, localfilesdir, extravalues, d)
711 lines_before.append("LICENSE += \" & ${GO_MOD_LICENSES}\"")
712 lines_before.append("require %s-licenses.inc" % (pn))
713
714 self.__rewrite_src_uri(lines_before, ["file://modules.txt"])
715
716 self.__go_handle_dependencies(go_mod, srctree, localfilesdir, extravalues, d)
717 lines_before.append("require %s-modules.inc" % (pn))
718
719 # Do generic license handling
720 handle_license_vars(srctree, lines_before, handled, extravalues, d)
721 self.__rewrite_lic_uri(lines_before)
722
723 lines_before.append("GO_IMPORT = \"{}\"".format(baseurl))
724 lines_before.append("SRCREV_FORMAT = \"${BPN}\"")
725
726 def __update_lines_before(self, updated, newlines, lines_before):
727 if updated:
728 del lines_before[:]
729 for line in newlines:
730 # Hack to avoid newlines that edit_metadata inserts
731 if line.endswith('\n'):
732 line = line[:-1]
733 lines_before.append(line)
734 return updated
735
736 def __rewrite_lic_uri(self, lines_before):
737
738 def varfunc(varname, origvalue, op, newlines):
739 if varname == 'LIC_FILES_CHKSUM':
740 new_licenses = []
741 licenses = origvalue.split('\\')
742 for license in licenses:
743 if not license:
744 logger.warning("No license file was detected for the main module!")
745 # the license list of the main recipe must be empty
746 # this can happen for example in case of CLOSED license
747 # Fall through to complete recipe generation
748 continue
749 license = license.strip()
750 uri, chksum = license.split(';', 1)
751 url = urllib.parse.urlparse(uri)
752 new_uri = os.path.join(
753 url.scheme + "://", "src", "${GO_IMPORT}", url.netloc + url.path) + ";" + chksum
754 new_licenses.append(new_uri)
755
756 return new_licenses, None, -1, True
757 return origvalue, None, 0, True
758
759 updated, newlines = bb.utils.edit_metadata(
760 lines_before, ['LIC_FILES_CHKSUM'], varfunc)
761 return self.__update_lines_before(updated, newlines, lines_before)
762
763 def __rewrite_src_uri(self, lines_before, additional_uris = []):
764
765 def varfunc(varname, origvalue, op, newlines):
766 if varname == 'SRC_URI':
767 src_uri = ["git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https"]
768 src_uri.extend(additional_uris)
769 return src_uri, None, -1, True
770 return origvalue, None, 0, True
771
772 updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
773 return self.__update_lines_before(updated, newlines, lines_before)
774
775
776def register_recipe_handlers(handlers):
777 handlers.append((GoRecipeHandler(), 60))
diff --git a/scripts/lib/recipetool/create_kmod.py b/scripts/lib/recipetool/create_kmod.py
index 85b5c48e53..cc00106961 100644
--- a/scripts/lib/recipetool/create_kmod.py
+++ b/scripts/lib/recipetool/create_kmod.py
@@ -113,7 +113,7 @@ class KernelModuleRecipeHandler(RecipeHandler):
113 kdirpath, _ = check_target(compile_lines, install=False) 113 kdirpath, _ = check_target(compile_lines, install=False)
114 114
115 if manual_install or not install_lines: 115 if manual_install or not install_lines:
116 lines_after.append('EXTRA_OEMAKE_append_task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"') 116 lines_after.append('EXTRA_OEMAKE:append:task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"')
117 elif install_target and install_target != 'modules_install': 117 elif install_target and install_target != 'modules_install':
118 lines_after.append('MODULES_INSTALL_TARGET = "install"') 118 lines_after.append('MODULES_INSTALL_TARGET = "install"')
119 119
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
index 2bcae91dfa..113a89f6a6 100644
--- a/scripts/lib/recipetool/create_npm.py
+++ b/scripts/lib/recipetool/create_npm.py
@@ -6,16 +6,20 @@
6"""Recipe creation tool - npm module support plugin""" 6"""Recipe creation tool - npm module support plugin"""
7 7
8import json 8import json
9import logging
9import os 10import os
10import re 11import re
11import sys 12import sys
12import tempfile 13import tempfile
13import bb 14import bb
14from bb.fetch2.npm import NpmEnvironment 15from bb.fetch2.npm import NpmEnvironment
16from bb.fetch2.npm import npm_package
15from bb.fetch2.npmsw import foreach_dependencies 17from bb.fetch2.npmsw import foreach_dependencies
16from recipetool.create import RecipeHandler 18from recipetool.create import RecipeHandler
19from recipetool.create import get_license_md5sums
17from recipetool.create import guess_license 20from recipetool.create import guess_license
18from recipetool.create import split_pkg_licenses 21from recipetool.create import split_pkg_licenses
22logger = logging.getLogger('recipetool')
19 23
20TINFOIL = None 24TINFOIL = None
21 25
@@ -28,15 +32,6 @@ class NpmRecipeHandler(RecipeHandler):
28 """Class to handle the npm recipe creation""" 32 """Class to handle the npm recipe creation"""
29 33
30 @staticmethod 34 @staticmethod
31 def _npm_name(name):
32 """Generate a Yocto friendly npm name"""
33 name = re.sub("/", "-", name)
34 name = name.lower()
35 name = re.sub(r"[^\-a-z0-9]", "", name)
36 name = name.strip("-")
37 return name
38
39 @staticmethod
40 def _get_registry(lines): 35 def _get_registry(lines):
41 """Get the registry value from the 'npm://registry' url""" 36 """Get the registry value from the 'npm://registry' url"""
42 registry = None 37 registry = None
@@ -118,23 +113,32 @@ class NpmRecipeHandler(RecipeHandler):
118 licfiles = [] 113 licfiles = []
119 packages = {} 114 packages = {}
120 115
121 def _licfiles_append(licfile):
122 """Append 'licfile' to the license files list"""
123 licfilepath = os.path.join(srctree, licfile)
124 licmd5 = bb.utils.md5_file(licfilepath)
125 licfiles.append("file://%s;md5=%s" % (licfile, licmd5))
126
127 # Handle the parent package 116 # Handle the parent package
128 _licfiles_append("package.json")
129 packages["${PN}"] = "" 117 packages["${PN}"] = ""
130 118
119 def _licfiles_append_fallback_readme_files(destdir):
120 """Append README files as fallback to license files if a license files is missing"""
121
122 fallback = True
123 readmes = []
124 basedir = os.path.join(srctree, destdir)
125 for fn in os.listdir(basedir):
126 upper = fn.upper()
127 if upper.startswith("README"):
128 fullpath = os.path.join(basedir, fn)
129 readmes.append(fullpath)
130 if upper.startswith("COPYING") or "LICENCE" in upper or "LICENSE" in upper:
131 fallback = False
132 if fallback:
133 for readme in readmes:
134 licfiles.append(os.path.relpath(readme, srctree))
135
131 # Handle the dependencies 136 # Handle the dependencies
132 def _handle_dependency(name, params, deptree): 137 def _handle_dependency(name, params, destdir):
133 suffix = "-".join([self._npm_name(dep) for dep in deptree]) 138 deptree = destdir.split('node_modules/')
134 destdirs = [os.path.join("node_modules", dep) for dep in deptree] 139 suffix = "-".join([npm_package(dep) for dep in deptree])
135 destdir = os.path.join(*destdirs) 140 packages["${PN}" + suffix] = destdir
136 _licfiles_append(os.path.join(destdir, "package.json")) 141 _licfiles_append_fallback_readme_files(destdir)
137 packages["${PN}-" + suffix] = destdir
138 142
139 with open(shrinkwrap_file, "r") as f: 143 with open(shrinkwrap_file, "r") as f:
140 shrinkwrap = json.load(f) 144 shrinkwrap = json.load(f)
@@ -142,6 +146,23 @@ class NpmRecipeHandler(RecipeHandler):
142 foreach_dependencies(shrinkwrap, _handle_dependency, dev) 146 foreach_dependencies(shrinkwrap, _handle_dependency, dev)
143 147
144 return licfiles, packages 148 return licfiles, packages
149
150 # Handle the peer dependencies
151 def _handle_peer_dependency(self, shrinkwrap_file):
152 """Check if package has peer dependencies and show warning if it is the case"""
153 with open(shrinkwrap_file, "r") as f:
154 shrinkwrap = json.load(f)
155
156 packages = shrinkwrap.get("packages", {})
157 peer_deps = packages.get("", {}).get("peerDependencies", {})
158
159 for peer_dep in peer_deps:
160 peer_dep_yocto_name = npm_package(peer_dep)
161 bb.warn(peer_dep + " is a peer dependencie of the actual package. " +
162 "Please add this peer dependencie to the RDEPENDS variable as %s and generate its recipe with devtool"
163 % peer_dep_yocto_name)
164
165
145 166
146 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): 167 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
147 """Handle the npm recipe creation""" 168 """Handle the npm recipe creation"""
@@ -160,7 +181,7 @@ class NpmRecipeHandler(RecipeHandler):
160 if "name" not in data or "version" not in data: 181 if "name" not in data or "version" not in data:
161 return False 182 return False
162 183
163 extravalues["PN"] = self._npm_name(data["name"]) 184 extravalues["PN"] = npm_package(data["name"])
164 extravalues["PV"] = data["version"] 185 extravalues["PV"] = data["version"]
165 186
166 if "description" in data: 187 if "description" in data:
@@ -229,7 +250,7 @@ class NpmRecipeHandler(RecipeHandler):
229 value = origvalue.replace("version=" + data["version"], "version=${PV}") 250 value = origvalue.replace("version=" + data["version"], "version=${PV}")
230 value = value.replace("version=latest", "version=${PV}") 251 value = value.replace("version=latest", "version=${PV}")
231 values = [line.strip() for line in value.strip('\n').splitlines()] 252 values = [line.strip() for line in value.strip('\n').splitlines()]
232 if "dependencies" in shrinkwrap: 253 if "dependencies" in shrinkwrap.get("packages", {}).get("", {}):
233 values.append(url_recipe) 254 values.append(url_recipe)
234 return values, None, 4, False 255 return values, None, 4, False
235 256
@@ -246,12 +267,42 @@ class NpmRecipeHandler(RecipeHandler):
246 267
247 bb.note("Handling licences ...") 268 bb.note("Handling licences ...")
248 (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev) 269 (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev)
249 extravalues["LIC_FILES_CHKSUM"] = licfiles 270
250 split_pkg_licenses(guess_license(srctree, d), packages, lines_after, []) 271 def _guess_odd_license(licfiles):
272 import bb
273
274 md5sums = get_license_md5sums(d, linenumbers=True)
275
276 chksums = []
277 licenses = []
278 for licfile in licfiles:
279 f = os.path.join(srctree, licfile)
280 md5value = bb.utils.md5_file(f)
281 (license, beginline, endline, md5) = md5sums.get(md5value,
282 (None, "", "", ""))
283 if not license:
284 license = "Unknown"
285 logger.info("Please add the following line for '%s' to a "
286 "'lib/recipetool/licenses.csv' and replace `Unknown`, "
287 "`X`, `Y` and `MD5` with the license, begin line, "
288 "end line and partial MD5 checksum:\n" \
289 "%s,Unknown,X,Y,MD5" % (licfile, md5value))
290 chksums.append("file://%s%s%s;md5=%s" % (licfile,
291 ";beginline=%s" % (beginline) if beginline else "",
292 ";endline=%s" % (endline) if endline else "",
293 md5 if md5 else md5value))
294 licenses.append((license, licfile, md5value))
295 return (licenses, chksums)
296
297 (licenses, extravalues["LIC_FILES_CHKSUM"]) = _guess_odd_license(licfiles)
298 split_pkg_licenses([*licenses, *guess_license(srctree, d)], packages, lines_after)
251 299
252 classes.append("npm") 300 classes.append("npm")
253 handled.append("buildsystem") 301 handled.append("buildsystem")
254 302
303 # Check if package has peer dependencies and inform the user
304 self._handle_peer_dependency(shrinkwrap_file)
305
255 return True 306 return True
256 307
257def register_recipe_handlers(handlers): 308def register_recipe_handlers(handlers):
diff --git a/scripts/lib/recipetool/licenses.csv b/scripts/lib/recipetool/licenses.csv
new file mode 100644
index 0000000000..80851111b3
--- /dev/null
+++ b/scripts/lib/recipetool/licenses.csv
@@ -0,0 +1,37 @@
10636e73ff0215e8d672dc4c32c317bb3,GPL-2.0-only
212f884d2ae1ff87c09e5b7ccc2c4ca7e,GPL-2.0-only
318810669f13b87348459e611d31ab760,GPL-2.0-only
4252890d9eee26aab7b432e8b8a616475,LGPL-2.0-only
52d5025d4aa3495befef8f17206a5b0a1,LGPL-2.1-only
63214f080875748938ba060314b4f727d,LGPL-2.0-only
7385c55653886acac3821999a3ccd17b3,Artistic-1.0 | GPL-2.0-only
8393a5ca445f6965873eca0259a17f833,GPL-2.0-only
93b83ef96387f14655fc854ddc3c6bd57,Apache-2.0
103bf50002aefd002f49e7bb854063f7e7,LGPL-2.0-only
114325afd396febcb659c36b49533135d4,GPL-2.0-only
124fbd65380cdd255951079008b364516c,LGPL-2.1-only
1354c7042be62e169199200bc6477f04d1,BSD-3-Clause
1455ca817ccb7d5b5b66355690e9abc605,LGPL-2.0-only
1559530bdf33659b29e73d4adb9f9f6552,GPL-2.0-only
165f30f0716dfdd0d91eb439ebec522ec2,LGPL-2.0-only
176a6a8e020838b23406c81b19c1d46df6,LGPL-3.0-only
18751419260aa954499f7abaabaa882bbe,GPL-2.0-only
197fbc338309ac38fefcd64b04bb903e34,LGPL-2.1-only
208ca43cbc842c2336e835926c2166c28b,GPL-2.0-only
2194d55d512a9ba36caa9b7df079bae19f,GPL-2.0-only
229ac2e7cff1ddaf48b6eab6028f23ef88,GPL-2.0-only
239f604d8a4f8e74f4f5140845a21b6674,LGPL-2.0-only
24a6f89e2100d9b6cdffcea4f398e37343,LGPL-2.1-only
25b234ee4d69f5fce4486a80fdaf4a4263,GPL-2.0-only
26bbb461211a33b134d42ed5ee802b37ff,LGPL-2.1-only
27bfe1f75d606912a4111c90743d6c7325,MPL-1.1-only
28c93c0550bd3173f4504b2cbd8991e50b,GPL-2.0-only
29d32239bcb673463ab874e80d47fae504,GPL-3.0-only
30d7810fab7487fb0aad327b76f1be7cd7,GPL-2.0-only
31d8045f3b8f929c1cb29a1e3fd737b499,LGPL-2.1-only
32db979804f025cf55aabec7129cb671ed,LGPL-2.0-only
33eb723b61539feef013de476e68b5c50a,GPL-2.0-only
34ebb5c50ab7cab4baeffba14977030c07,GPL-2.0-only
35f27defe1e96c2e1ecd4e0c9be8967949,GPL-3.0-only
36fad9b3332be894bab9bc501572864b29,LGPL-2.1-only
37fbc093901857fcd118f065f900982c24,LGPL-2.1-only
diff --git a/scripts/lib/recipetool/setvar.py b/scripts/lib/recipetool/setvar.py
index f8e2ee75fb..b5ad335cae 100644
--- a/scripts/lib/recipetool/setvar.py
+++ b/scripts/lib/recipetool/setvar.py
@@ -49,6 +49,7 @@ def setvar(args):
49 for patch in patches: 49 for patch in patches:
50 for line in patch: 50 for line in patch:
51 sys.stdout.write(line) 51 sys.stdout.write(line)
52 tinfoil.modified_files()
52 return 0 53 return 0
53 54
54 55
diff --git a/scripts/lib/resulttool/log.py b/scripts/lib/resulttool/log.py
index eb3927ec82..15148ca288 100644
--- a/scripts/lib/resulttool/log.py
+++ b/scripts/lib/resulttool/log.py
@@ -28,12 +28,10 @@ def show_reproducible(result, reproducible, logger):
28def log(args, logger): 28def log(args, logger):
29 results = resultutils.load_resultsdata(args.source) 29 results = resultutils.load_resultsdata(args.source)
30 30
31 ptest_count = sum(1 for _, _, _, r in resultutils.test_run_results(results) if 'ptestresult.sections' in r)
32 if ptest_count > 1 and not args.prepend_run:
33 print("%i ptest sections found. '--prepend-run' is required" % ptest_count)
34 return 1
35
36 for _, run_name, _, r in resultutils.test_run_results(results): 31 for _, run_name, _, r in resultutils.test_run_results(results):
32 if args.list_ptest:
33 print('\n'.join(sorted(r['ptestresult.sections'].keys())))
34
37 if args.dump_ptest: 35 if args.dump_ptest:
38 for sectname in ['ptestresult.sections', 'ltpposixresult.sections', 'ltpresult.sections']: 36 for sectname in ['ptestresult.sections', 'ltpposixresult.sections', 'ltpresult.sections']:
39 if sectname in r: 37 if sectname in r:
@@ -48,6 +46,9 @@ def log(args, logger):
48 46
49 os.makedirs(dest_dir, exist_ok=True) 47 os.makedirs(dest_dir, exist_ok=True)
50 dest = os.path.join(dest_dir, '%s.log' % name) 48 dest = os.path.join(dest_dir, '%s.log' % name)
49 if os.path.exists(dest):
50 print("Overlapping ptest logs found, skipping %s. The '--prepend-run' option would avoid this" % name)
51 continue
51 print(dest) 52 print(dest)
52 with open(dest, 'w') as f: 53 with open(dest, 'w') as f:
53 f.write(logdata) 54 f.write(logdata)
@@ -86,6 +87,8 @@ def register_commands(subparsers):
86 parser.set_defaults(func=log) 87 parser.set_defaults(func=log)
87 parser.add_argument('source', 88 parser.add_argument('source',
88 help='the results file/directory/URL to import') 89 help='the results file/directory/URL to import')
90 parser.add_argument('--list-ptest', action='store_true',
91 help='list the ptest test names')
89 parser.add_argument('--ptest', action='append', default=[], 92 parser.add_argument('--ptest', action='append', default=[],
90 help='show logs for a ptest') 93 help='show logs for a ptest')
91 parser.add_argument('--dump-ptest', metavar='DIR', 94 parser.add_argument('--dump-ptest', metavar='DIR',
diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py
index 9f952951b3..10e7d13841 100644
--- a/scripts/lib/resulttool/regression.py
+++ b/scripts/lib/resulttool/regression.py
@@ -7,15 +7,209 @@
7# 7#
8 8
9import resulttool.resultutils as resultutils 9import resulttool.resultutils as resultutils
10import json
11 10
12from oeqa.utils.git import GitRepo 11from oeqa.utils.git import GitRepo
13import oeqa.utils.gitarchive as gitarchive 12import oeqa.utils.gitarchive as gitarchive
14 13
15def compare_result(logger, base_name, target_name, base_result, target_result): 14METADATA_MATCH_TABLE = {
15 "oeselftest": "OESELFTEST_METADATA"
16}
17
18OESELFTEST_METADATA_GUESS_TABLE={
19 "trigger-build-posttrigger": {
20 "run_all_tests": False,
21 "run_tests":["buildoptions.SourceMirroring.test_yocto_source_mirror"],
22 "skips": None,
23 "machine": None,
24 "select_tags":None,
25 "exclude_tags": None
26 },
27 "reproducible": {
28 "run_all_tests": False,
29 "run_tests":["reproducible"],
30 "skips": None,
31 "machine": None,
32 "select_tags":None,
33 "exclude_tags": None
34 },
35 "arch-qemu-quick": {
36 "run_all_tests": True,
37 "run_tests":None,
38 "skips": None,
39 "machine": None,
40 "select_tags":["machine"],
41 "exclude_tags": None
42 },
43 "arch-qemu-full-x86-or-x86_64": {
44 "run_all_tests": True,
45 "run_tests":None,
46 "skips": None,
47 "machine": None,
48 "select_tags":["machine", "toolchain-system"],
49 "exclude_tags": None
50 },
51 "arch-qemu-full-others": {
52 "run_all_tests": True,
53 "run_tests":None,
54 "skips": None,
55 "machine": None,
56 "select_tags":["machine", "toolchain-user"],
57 "exclude_tags": None
58 },
59 "selftest": {
60 "run_all_tests": True,
61 "run_tests":None,
62 "skips": ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror", "reproducible"],
63 "machine": None,
64 "select_tags":None,
65 "exclude_tags": ["machine", "toolchain-system", "toolchain-user"]
66 },
67 "bringup": {
68 "run_all_tests": True,
69 "run_tests":None,
70 "skips": ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror"],
71 "machine": None,
72 "select_tags":None,
73 "exclude_tags": ["machine", "toolchain-system", "toolchain-user"]
74 }
75}
76
77STATUS_STRINGS = {
78 "None": "No matching test result"
79}
80
81REGRESSIONS_DISPLAY_LIMIT=50
82
83MISSING_TESTS_BANNER = "-------------------------- Missing tests --------------------------"
84ADDITIONAL_DATA_BANNER = "--------------------- Matches and improvements --------------------"
85
86def test_has_at_least_one_matching_tag(test, tag_list):
87 return "oetags" in test and any(oetag in tag_list for oetag in test["oetags"])
88
89def all_tests_have_at_least_one_matching_tag(results, tag_list):
90 return all(test_has_at_least_one_matching_tag(test_result, tag_list) or test_name.startswith("ptestresult") for (test_name, test_result) in results.items())
91
92def any_test_have_any_matching_tag(results, tag_list):
93 return any(test_has_at_least_one_matching_tag(test, tag_list) for test in results.values())
94
95def have_skipped_test(result, test_prefix):
96 return all( result[test]['status'] == "SKIPPED" for test in result if test.startswith(test_prefix))
97
98def have_all_tests_skipped(result, test_prefixes_list):
99 return all(have_skipped_test(result, test_prefix) for test_prefix in test_prefixes_list)
100
101def guess_oeselftest_metadata(results):
102 """
103 When an oeselftest test result is lacking OESELFTEST_METADATA, we can try to guess it based on results content.
104 Check results for specific values (absence/presence of oetags, number and name of executed tests...),
105 and if it matches one of known configuration from autobuilder configuration, apply guessed OSELFTEST_METADATA
106 to it to allow proper test filtering.
107 This guessing process is tightly coupled to config.json in autobuilder. It should trigger less and less,
108 as new tests will have OESELFTEST_METADATA properly appended at test reporting time
109 """
110
111 if len(results) == 1 and "buildoptions.SourceMirroring.test_yocto_source_mirror" in results:
112 return OESELFTEST_METADATA_GUESS_TABLE['trigger-build-posttrigger']
113 elif all(result.startswith("reproducible") for result in results):
114 return OESELFTEST_METADATA_GUESS_TABLE['reproducible']
115 elif all_tests_have_at_least_one_matching_tag(results, ["machine"]):
116 return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-quick']
117 elif all_tests_have_at_least_one_matching_tag(results, ["machine", "toolchain-system"]):
118 return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-full-x86-or-x86_64']
119 elif all_tests_have_at_least_one_matching_tag(results, ["machine", "toolchain-user"]):
120 return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-full-others']
121 elif not any_test_have_any_matching_tag(results, ["machine", "toolchain-user", "toolchain-system"]):
122 if have_all_tests_skipped(results, ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror", "reproducible"]):
123 return OESELFTEST_METADATA_GUESS_TABLE['selftest']
124 elif have_all_tests_skipped(results, ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror"]):
125 return OESELFTEST_METADATA_GUESS_TABLE['bringup']
126
127 return None
128
129
130def metadata_matches(base_configuration, target_configuration):
131 """
132 For passed base and target, check test type. If test type matches one of
133 properties described in METADATA_MATCH_TABLE, compare metadata if it is
134 present in base. Return true if metadata matches, or if base lacks some
135 data (either TEST_TYPE or the corresponding metadata)
136 """
137 test_type = base_configuration.get('TEST_TYPE')
138 if test_type not in METADATA_MATCH_TABLE:
139 return True
140
141 metadata_key = METADATA_MATCH_TABLE.get(test_type)
142 if target_configuration.get(metadata_key) != base_configuration.get(metadata_key):
143 return False
144
145 return True
146
147
148def machine_matches(base_configuration, target_configuration):
149 return base_configuration.get('MACHINE') == target_configuration.get('MACHINE')
150
151
152def can_be_compared(logger, base, target):
153 """
154 Some tests are not relevant to be compared, for example some oeselftest
155 run with different tests sets or parameters. Return true if tests can be
156 compared
157 """
158 ret = True
159 base_configuration = base['configuration']
160 target_configuration = target['configuration']
161
162 # Older test results lack proper OESELFTEST_METADATA: if not present, try to guess it based on tests results.
163 if base_configuration.get('TEST_TYPE') == 'oeselftest' and 'OESELFTEST_METADATA' not in base_configuration:
164 guess = guess_oeselftest_metadata(base['result'])
165 if guess is None:
166 logger.error(f"ERROR: did not manage to guess oeselftest metadata for {base_configuration['STARTTIME']}")
167 else:
168 logger.debug(f"Enriching {base_configuration['STARTTIME']} with {guess}")
169 base_configuration['OESELFTEST_METADATA'] = guess
170 if target_configuration.get('TEST_TYPE') == 'oeselftest' and 'OESELFTEST_METADATA' not in target_configuration:
171 guess = guess_oeselftest_metadata(target['result'])
172 if guess is None:
173 logger.error(f"ERROR: did not manage to guess oeselftest metadata for {target_configuration['STARTTIME']}")
174 else:
175 logger.debug(f"Enriching {target_configuration['STARTTIME']} with {guess}")
176 target_configuration['OESELFTEST_METADATA'] = guess
177
178 # Test runs with LTP results in should only be compared with other runs with LTP tests in them
179 if base_configuration.get('TEST_TYPE') == 'runtime' and any(result.startswith("ltpresult") for result in base['result']):
180 ret = target_configuration.get('TEST_TYPE') == 'runtime' and any(result.startswith("ltpresult") for result in target['result'])
181
182 return ret and metadata_matches(base_configuration, target_configuration) \
183 and machine_matches(base_configuration, target_configuration)
184
185def get_status_str(raw_status):
186 raw_status_lower = raw_status.lower() if raw_status else "None"
187 return STATUS_STRINGS.get(raw_status_lower, raw_status)
188
189def get_additional_info_line(new_pass_count, new_tests):
190 result=[]
191 if new_tests:
192 result.append(f'+{new_tests} test(s) present')
193 if new_pass_count:
194 result.append(f'+{new_pass_count} test(s) now passing')
195
196 if not result:
197 return ""
198
199 return ' -> ' + ', '.join(result) + '\n'
200
201def compare_result(logger, base_name, target_name, base_result, target_result, display_limit=None):
16 base_result = base_result.get('result') 202 base_result = base_result.get('result')
17 target_result = target_result.get('result') 203 target_result = target_result.get('result')
18 result = {} 204 result = {}
205 new_tests = 0
206 regressions = {}
207 resultstring = ""
208 new_tests = 0
209 new_pass_count = 0
210
211 display_limit = int(display_limit) if display_limit else REGRESSIONS_DISPLAY_LIMIT
212
19 if base_result and target_result: 213 if base_result and target_result:
20 for k in base_result: 214 for k in base_result:
21 base_testcase = base_result[k] 215 base_testcase = base_result[k]
@@ -27,12 +221,47 @@ def compare_result(logger, base_name, target_name, base_result, target_result):
27 result[k] = {'base': base_status, 'target': target_status} 221 result[k] = {'base': base_status, 'target': target_status}
28 else: 222 else:
29 logger.error('Failed to retrieved base test case status: %s' % k) 223 logger.error('Failed to retrieved base test case status: %s' % k)
224
225 # Also count new tests that were not present in base results: it
226 # could be newly added tests, but it could also highlights some tests
227 # renames or fixed faulty ptests
228 for k in target_result:
229 if k not in base_result:
230 new_tests += 1
30 if result: 231 if result:
31 resultstring = "Regression: %s\n %s\n" % (base_name, target_name) 232 new_pass_count = sum(test['target'] is not None and test['target'].startswith("PASS") for test in result.values())
32 for k in sorted(result): 233 # Print a regression report only if at least one test has a regression status (FAIL, SKIPPED, absent...)
33 resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target']) 234 if new_pass_count < len(result):
235 resultstring = "Regression: %s\n %s\n" % (base_name, target_name)
236 for k in sorted(result):
237 if not result[k]['target'] or not result[k]['target'].startswith("PASS"):
238 # Differentiate each ptest kind when listing regressions
239 key_parts = k.split('.')
240 key = '.'.join(key_parts[:2]) if k.startswith('ptest') else key_parts[0]
241 # Append new regression to corresponding test family
242 regressions[key] = regressions.setdefault(key, []) + [' %s: %s -> %s\n' % (k, get_status_str(result[k]['base']), get_status_str(result[k]['target']))]
243 resultstring += f" Total: {sum([len(regressions[r]) for r in regressions])} new regression(s):\n"
244 for k in regressions:
245 resultstring += f" {len(regressions[k])} regression(s) for {k}\n"
246 count_to_print=min([display_limit, len(regressions[k])]) if display_limit > 0 else len(regressions[k])
247 resultstring += ''.join(regressions[k][:count_to_print])
248 if count_to_print < len(regressions[k]):
249 resultstring+=' [...]\n'
250 if new_pass_count > 0:
251 resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n'
252 if new_tests > 0:
253 resultstring += f' Additionally, {new_tests} new test(s) is/are present\n'
254 else:
255 resultstring = "%s\n%s\n" % (base_name, target_name)
256 result = None
34 else: 257 else:
35 resultstring = "Match: %s\n %s" % (base_name, target_name) 258 resultstring = "%s\n%s\n" % (base_name, target_name)
259
260 if not result:
261 additional_info = get_additional_info_line(new_pass_count, new_tests)
262 if additional_info:
263 resultstring += additional_info
264
36 return result, resultstring 265 return result, resultstring
37 266
38def get_results(logger, source): 267def get_results(logger, source):
@@ -44,12 +273,38 @@ def regression(args, logger):
44 273
45 regression_common(args, logger, base_results, target_results) 274 regression_common(args, logger, base_results, target_results)
46 275
276# Some test case naming is poor and contains random strings, particularly lttng/babeltrace.
277# Truncating the test names works since they contain file and line number identifiers
278# which allows us to match them without the random components.
279def fixup_ptest_names(results, logger):
280 for r in results:
281 for i in results[r]:
282 tests = list(results[r][i]['result'].keys())
283 for test in tests:
284 new = None
285 if test.startswith(("ptestresult.lttng-tools.", "ptestresult.babeltrace.", "ptestresult.babeltrace2")) and "_-_" in test:
286 new = test.split("_-_")[0]
287 elif test.startswith(("ptestresult.curl.")) and "__" in test:
288 new = test.split("__")[0]
289 elif test.startswith(("ptestresult.dbus.")) and "__" in test:
290 new = test.split("__")[0]
291 elif test.startswith("ptestresult.binutils") and "build-st-" in test:
292 new = test.split(" ")[0]
293 elif test.startswith("ptestresult.gcc") and "/tmp/runtest." in test:
294 new = ".".join(test.split(".")[:2])
295 if new:
296 results[r][i]['result'][new] = results[r][i]['result'][test]
297 del results[r][i]['result'][test]
298
47def regression_common(args, logger, base_results, target_results): 299def regression_common(args, logger, base_results, target_results):
48 if args.base_result_id: 300 if args.base_result_id:
49 base_results = resultutils.filter_resultsdata(base_results, args.base_result_id) 301 base_results = resultutils.filter_resultsdata(base_results, args.base_result_id)
50 if args.target_result_id: 302 if args.target_result_id:
51 target_results = resultutils.filter_resultsdata(target_results, args.target_result_id) 303 target_results = resultutils.filter_resultsdata(target_results, args.target_result_id)
52 304
305 fixup_ptest_names(base_results, logger)
306 fixup_ptest_names(target_results, logger)
307
53 matches = [] 308 matches = []
54 regressions = [] 309 regressions = []
55 notfound = [] 310 notfound = []
@@ -62,7 +317,9 @@ def regression_common(args, logger, base_results, target_results):
62 # removing any pairs which match 317 # removing any pairs which match
63 for c in base.copy(): 318 for c in base.copy():
64 for b in target.copy(): 319 for b in target.copy():
65 res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) 320 if not can_be_compared(logger, base_results[a][c], target_results[a][b]):
321 continue
322 res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b], args.limit)
66 if not res: 323 if not res:
67 matches.append(resstr) 324 matches.append(resstr)
68 base.remove(c) 325 base.remove(c)
@@ -71,15 +328,18 @@ def regression_common(args, logger, base_results, target_results):
71 # Should only now see regressions, we may not be able to match multiple pairs directly 328 # Should only now see regressions, we may not be able to match multiple pairs directly
72 for c in base: 329 for c in base:
73 for b in target: 330 for b in target:
74 res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) 331 if not can_be_compared(logger, base_results[a][c], target_results[a][b]):
332 continue
333 res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b], args.limit)
75 if res: 334 if res:
76 regressions.append(resstr) 335 regressions.append(resstr)
77 else: 336 else:
78 notfound.append("%s not found in target" % a) 337 notfound.append("%s not found in target" % a)
79 print("\n".join(sorted(matches)))
80 print("\n".join(sorted(regressions))) 338 print("\n".join(sorted(regressions)))
339 print("\n" + MISSING_TESTS_BANNER + "\n")
81 print("\n".join(sorted(notfound))) 340 print("\n".join(sorted(notfound)))
82 341 print("\n" + ADDITIONAL_DATA_BANNER + "\n")
342 print("\n".join(sorted(matches)))
83 return 0 343 return 0
84 344
85def regression_git(args, logger): 345def regression_git(args, logger):
@@ -183,4 +443,5 @@ def register_commands(subparsers):
183 parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified") 443 parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified")
184 parser_build.add_argument('--commit2', help="Revision to compare with") 444 parser_build.add_argument('--commit2', help="Revision to compare with")
185 parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified") 445 parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified")
446 parser_build.add_argument('-l', '--limit', default=REGRESSIONS_DISPLAY_LIMIT, help="Maximum number of changes to display per test. Can be set to 0 to print all changes")
186 447
diff --git a/scripts/lib/resulttool/report.py b/scripts/lib/resulttool/report.py
index f0ca50ebe2..a349510ab8 100644
--- a/scripts/lib/resulttool/report.py
+++ b/scripts/lib/resulttool/report.py
@@ -176,7 +176,10 @@ class ResultsTextReport(object):
176 vals['sort'] = line['testseries'] + "_" + line['result_id'] 176 vals['sort'] = line['testseries'] + "_" + line['result_id']
177 vals['failed_testcases'] = line['failed_testcases'] 177 vals['failed_testcases'] = line['failed_testcases']
178 for k in cols: 178 for k in cols:
179 vals[k] = "%d (%s%%)" % (line[k], format(line[k] / total_tested * 100, '.0f')) 179 if total_tested:
180 vals[k] = "%d (%s%%)" % (line[k], format(line[k] / total_tested * 100, '.0f'))
181 else:
182 vals[k] = "0 (0%)"
180 for k in maxlen: 183 for k in maxlen:
181 if k in vals and len(vals[k]) > maxlen[k]: 184 if k in vals and len(vals[k]) > maxlen[k]:
182 maxlen[k] = len(vals[k]) 185 maxlen[k] = len(vals[k])
diff --git a/scripts/lib/resulttool/resultutils.py b/scripts/lib/resulttool/resultutils.py
index 8917022d36..c5521d81bd 100644
--- a/scripts/lib/resulttool/resultutils.py
+++ b/scripts/lib/resulttool/resultutils.py
@@ -58,7 +58,11 @@ def append_resultsdata(results, f, configmap=store_map, configvars=extra_configv
58 testseries = posixpath.basename(posixpath.dirname(url.path)) 58 testseries = posixpath.basename(posixpath.dirname(url.path))
59 else: 59 else:
60 with open(f, "r") as filedata: 60 with open(f, "r") as filedata:
61 data = json.load(filedata) 61 try:
62 data = json.load(filedata)
63 except json.decoder.JSONDecodeError:
64 print("Cannot decode {}. Possible corruption. Skipping.".format(f))
65 data = ""
62 testseries = os.path.basename(os.path.dirname(f)) 66 testseries = os.path.basename(os.path.dirname(f))
63 else: 67 else:
64 data = f 68 data = f
@@ -142,7 +146,7 @@ def generic_get_log(sectionname, results, section):
142 return decode_log(ptest['log']) 146 return decode_log(ptest['log'])
143 147
144def ptestresult_get_log(results, section): 148def ptestresult_get_log(results, section):
145 return generic_get_log('ptestresuls.sections', results, section) 149 return generic_get_log('ptestresult.sections', results, section)
146 150
147def generic_get_rawlogs(sectname, results): 151def generic_get_rawlogs(sectname, results):
148 if sectname not in results: 152 if sectname not in results:
diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py
index f92255d8dc..f23e53cba9 100644
--- a/scripts/lib/scriptutils.py
+++ b/scripts/lib/scriptutils.py
@@ -5,7 +5,6 @@
5# SPDX-License-Identifier: GPL-2.0-only 5# SPDX-License-Identifier: GPL-2.0-only
6# 6#
7 7
8import argparse
9import glob 8import glob
10import logging 9import logging
11import os 10import os
@@ -18,13 +17,14 @@ import sys
18import tempfile 17import tempfile
19import threading 18import threading
20import importlib 19import importlib
21from importlib import machinery 20import importlib.machinery
21import importlib.util
22 22
23class KeepAliveStreamHandler(logging.StreamHandler): 23class KeepAliveStreamHandler(logging.StreamHandler):
24 def __init__(self, keepalive=True, **kwargs): 24 def __init__(self, keepalive=True, **kwargs):
25 super().__init__(**kwargs) 25 super().__init__(**kwargs)
26 if keepalive is True: 26 if keepalive is True:
27 keepalive = 5000 # default timeout 27 keepalive = 5000 # default timeout
28 self._timeout = threading.Condition() 28 self._timeout = threading.Condition()
29 self._stop = False 29 self._stop = False
30 30
@@ -35,9 +35,9 @@ class KeepAliveStreamHandler(logging.StreamHandler):
35 with self._timeout: 35 with self._timeout:
36 if not self._timeout.wait(keepalive): 36 if not self._timeout.wait(keepalive):
37 self.emit(logging.LogRecord("keepalive", logging.INFO, 37 self.emit(logging.LogRecord("keepalive", logging.INFO,
38 None, None, "Keepalive message", None, None)) 38 None, None, "Keepalive message", None, None))
39 39
40 self._thread = threading.Thread(target = thread, daemon = True) 40 self._thread = threading.Thread(target=thread, daemon=True)
41 self._thread.start() 41 self._thread.start()
42 42
43 def close(self): 43 def close(self):
@@ -71,18 +71,19 @@ def logger_setup_color(logger, color='auto'):
71 71
72 for handler in logger.handlers: 72 for handler in logger.handlers:
73 if (isinstance(handler, logging.StreamHandler) and 73 if (isinstance(handler, logging.StreamHandler) and
74 isinstance(handler.formatter, BBLogFormatter)): 74 isinstance(handler.formatter, BBLogFormatter)):
75 if color == 'always' or (color == 'auto' and handler.stream.isatty()): 75 if color == 'always' or (color == 'auto' and handler.stream.isatty()):
76 handler.formatter.enable_color() 76 handler.formatter.enable_color()
77 77
78 78
79def load_plugins(logger, plugins, pluginpath): 79def load_plugins(logger, plugins, pluginpath):
80
81 def load_plugin(name): 80 def load_plugin(name):
82 logger.debug('Loading plugin %s' % name) 81 logger.debug('Loading plugin %s' % name)
83 spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] ) 82 spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath])
84 if spec: 83 if spec:
85 return spec.loader.load_module() 84 mod = importlib.util.module_from_spec(spec)
85 spec.loader.exec_module(mod)
86 return mod
86 87
87 def plugin_name(filename): 88 def plugin_name(filename):
88 return os.path.splitext(os.path.basename(filename))[0] 89 return os.path.splitext(os.path.basename(filename))[0]
@@ -176,6 +177,7 @@ def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirr
176 f.write('BB_STRICT_CHECKSUM = "ignore"\n') 177 f.write('BB_STRICT_CHECKSUM = "ignore"\n')
177 f.write('SRC_URI = "%s"\n' % srcuri) 178 f.write('SRC_URI = "%s"\n' % srcuri)
178 f.write('SRCREV = "%s"\n' % srcrev) 179 f.write('SRCREV = "%s"\n' % srcrev)
180 f.write('PV = "0.0+"\n')
179 f.write('WORKDIR = "%s"\n' % tmpworkdir) 181 f.write('WORKDIR = "%s"\n' % tmpworkdir)
180 # Set S out of the way so it doesn't get created under the workdir 182 # Set S out of the way so it doesn't get created under the workdir
181 f.write('S = "%s"\n' % os.path.join(tmpdir, 'emptysrc')) 183 f.write('S = "%s"\n' % os.path.join(tmpdir, 'emptysrc'))
@@ -215,7 +217,8 @@ def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirr
215 pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE'] 217 pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE']
216 for pathvar in pathvars: 218 for pathvar in pathvars:
217 path = rd.getVar(pathvar) 219 path = rd.getVar(pathvar)
218 shutil.rmtree(path) 220 if os.path.exists(path):
221 shutil.rmtree(path)
219 finally: 222 finally:
220 if fetchrecipe: 223 if fetchrecipe:
221 try: 224 try:
@@ -274,6 +277,6 @@ def filter_src_subdirs(pth):
274 Used by devtool and recipetool. 277 Used by devtool and recipetool.
275 """ 278 """
276 dirlist = os.listdir(pth) 279 dirlist = os.listdir(pth)
277 filterout = ['git.indirectionsymlink', 'source-date-epoch'] 280 filterout = ['git.indirectionsymlink', 'source-date-epoch', 'sstate-install-recipe_qa']
278 dirlist = [x for x in dirlist if x not in filterout] 281 dirlist = [x for x in dirlist if x not in filterout]
279 return dirlist 282 return dirlist
diff --git a/scripts/lib/wic/canned-wks/common.wks.inc b/scripts/lib/wic/canned-wks/common.wks.inc
index 4fd29fa8c1..89880b417b 100644
--- a/scripts/lib/wic/canned-wks/common.wks.inc
+++ b/scripts/lib/wic/canned-wks/common.wks.inc
@@ -1,3 +1,3 @@
1# This file is included into 3 canned wks files from this directory 1# This file is included into 3 canned wks files from this directory
2part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024 2part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
3part / --source rootfs --use-uuid --fstype=ext4 --mkfs-extraopts "-T default" --label platform --align 1024 3part / --source rootfs --use-uuid --fstype=ext4 --label platform --align 1024
diff --git a/scripts/lib/wic/canned-wks/directdisk-gpt.wks b/scripts/lib/wic/canned-wks/directdisk-gpt.wks
index cf16c0c30b..8d7d8de6ea 100644
--- a/scripts/lib/wic/canned-wks/directdisk-gpt.wks
+++ b/scripts/lib/wic/canned-wks/directdisk-gpt.wks
@@ -4,7 +4,7 @@
4 4
5 5
6part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024 6part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
7part / --source rootfs --ondisk sda --fstype=ext4 --mkfs-extraopts "-T default" --label platform --align 1024 --use-uuid 7part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
8 8
9bootloader --ptable gpt --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8" 9bootloader --ptable gpt --timeout=0 --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8"
10 10
diff --git a/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in b/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in
index 7300e65e32..2fd286ff98 100644
--- a/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in
+++ b/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in
@@ -1,3 +1,3 @@
1bootloader --ptable gpt 1bootloader --ptable gpt
2part /boot --source rootfs --rootfs-dir=${IMAGE_ROOTFS}/boot --fstype=vfat --label boot --active --align 1024 --use-uuid --overhead-factor 1.0 2part /boot --source rootfs --rootfs-dir=${IMAGE_ROOTFS}/boot --fstype=vfat --label boot --active --align 1024 --use-uuid --overhead-factor 1.1
3part / --source rootfs --fstype=ext4 --label root --align 1024 --exclude-path boot/ 3part / --source rootfs --fstype=ext4 --label root --align 1024 --exclude-path boot/
diff --git a/scripts/lib/wic/canned-wks/mkefidisk.wks b/scripts/lib/wic/canned-wks/mkefidisk.wks
index d1878e23e5..9f534fe184 100644
--- a/scripts/lib/wic/canned-wks/mkefidisk.wks
+++ b/scripts/lib/wic/canned-wks/mkefidisk.wks
@@ -4,7 +4,7 @@
4 4
5part /boot --source bootimg-efi --sourceparams="loader=grub-efi" --ondisk sda --label msdos --active --align 1024 5part /boot --source bootimg-efi --sourceparams="loader=grub-efi" --ondisk sda --label msdos --active --align 1024
6 6
7part / --source rootfs --ondisk sda --fstype=ext4 --mkfs-extraopts "-T default" --label platform --align 1024 --use-uuid 7part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
8 8
9part swap --ondisk sda --size 44 --label swap1 --fstype=swap 9part swap --ondisk sda --size 44 --label swap1 --fstype=swap
10 10
diff --git a/scripts/lib/wic/canned-wks/qemuloongarch.wks b/scripts/lib/wic/canned-wks/qemuloongarch.wks
new file mode 100644
index 0000000000..8465c7a8c0
--- /dev/null
+++ b/scripts/lib/wic/canned-wks/qemuloongarch.wks
@@ -0,0 +1,3 @@
1# short-description: Create qcow2 image for LoongArch QEMU machines
2
3part / --source rootfs --fstype=ext4 --label root --align 4096 --size 5G
diff --git a/scripts/lib/wic/canned-wks/qemux86-directdisk.wks b/scripts/lib/wic/canned-wks/qemux86-directdisk.wks
index 22b45217f1..808997611a 100644
--- a/scripts/lib/wic/canned-wks/qemux86-directdisk.wks
+++ b/scripts/lib/wic/canned-wks/qemux86-directdisk.wks
@@ -4,5 +4,5 @@
4 4
5include common.wks.inc 5include common.wks.inc
6 6
7bootloader --timeout=0 --append="rw oprofile.timer=1 rootfstype=ext4 " 7bootloader --timeout=0 --append="rw oprofile.timer=1 rootfstype=ext4 console=tty console=ttyS0 "
8 8
diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
index 018815b966..674ccfc244 100644
--- a/scripts/lib/wic/engine.py
+++ b/scripts/lib/wic/engine.py
@@ -19,10 +19,10 @@ import os
19import tempfile 19import tempfile
20import json 20import json
21import subprocess 21import subprocess
22import shutil
22import re 23import re
23 24
24from collections import namedtuple, OrderedDict 25from collections import namedtuple, OrderedDict
25from distutils.spawn import find_executable
26 26
27from wic import WicError 27from wic import WicError
28from wic.filemap import sparse_copy 28from wic.filemap import sparse_copy
@@ -245,7 +245,7 @@ class Disk:
245 for path in pathlist.split(':'): 245 for path in pathlist.split(':'):
246 self.paths = "%s%s:%s" % (native_sysroot, path, self.paths) 246 self.paths = "%s%s:%s" % (native_sysroot, path, self.paths)
247 247
248 self.parted = find_executable("parted", self.paths) 248 self.parted = shutil.which("parted", path=self.paths)
249 if not self.parted: 249 if not self.parted:
250 raise WicError("Can't find executable parted") 250 raise WicError("Can't find executable parted")
251 251
@@ -283,7 +283,7 @@ class Disk:
283 "resize2fs", "mkswap", "mkdosfs", "debugfs","blkid"): 283 "resize2fs", "mkswap", "mkdosfs", "debugfs","blkid"):
284 aname = "_%s" % name 284 aname = "_%s" % name
285 if aname not in self.__dict__: 285 if aname not in self.__dict__:
286 setattr(self, aname, find_executable(name, self.paths)) 286 setattr(self, aname, shutil.which(name, path=self.paths))
287 if aname not in self.__dict__ or self.__dict__[aname] is None: 287 if aname not in self.__dict__ or self.__dict__[aname] is None:
288 raise WicError("Can't find executable '{}'".format(name)) 288 raise WicError("Can't find executable '{}'".format(name))
289 return self.__dict__[aname] 289 return self.__dict__[aname]
diff --git a/scripts/lib/wic/filemap.py b/scripts/lib/wic/filemap.py
index 4d9da28172..85b39d5d74 100644
--- a/scripts/lib/wic/filemap.py
+++ b/scripts/lib/wic/filemap.py
@@ -46,6 +46,13 @@ def get_block_size(file_obj):
46 bsize = stat.st_blksize 46 bsize = stat.st_blksize
47 else: 47 else:
48 raise IOError("Unable to determine block size") 48 raise IOError("Unable to determine block size")
49
50 # The logic in this script only supports a maximum of a 4KB
51 # block size
52 max_block_size = 4 * 1024
53 if bsize > max_block_size:
54 bsize = max_block_size
55
49 return bsize 56 return bsize
50 57
51class ErrorNotSupp(Exception): 58class ErrorNotSupp(Exception):
diff --git a/scripts/lib/wic/help.py b/scripts/lib/wic/help.py
index bd3a2b97df..163535e431 100644
--- a/scripts/lib/wic/help.py
+++ b/scripts/lib/wic/help.py
@@ -637,7 +637,7 @@ DESCRIPTION
637 oe-core: directdisk.bbclass and mkefidisk.sh. The difference 637 oe-core: directdisk.bbclass and mkefidisk.sh. The difference
638 between wic and those examples is that with wic the functionality 638 between wic and those examples is that with wic the functionality
639 of those scripts is implemented by a general-purpose partitioning 639 of those scripts is implemented by a general-purpose partitioning
640 'language' based on Redhat kickstart syntax). 640 'language' based on Red Hat kickstart syntax).
641 641
642 The initial motivation and design considerations that lead to the 642 The initial motivation and design considerations that lead to the
643 current tool are described exhaustively in Yocto Bug #3847 643 current tool are described exhaustively in Yocto Bug #3847
@@ -840,8 +840,8 @@ DESCRIPTION
840 meanings. The commands are based on the Fedora kickstart 840 meanings. The commands are based on the Fedora kickstart
841 documentation but with modifications to reflect wic capabilities. 841 documentation but with modifications to reflect wic capabilities.
842 842
843 http://fedoraproject.org/wiki/Anaconda/Kickstart#part_or_partition 843 https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#part-or-partition
844 http://fedoraproject.org/wiki/Anaconda/Kickstart#bootloader 844 https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#bootloader
845 845
846 Commands 846 Commands
847 847
@@ -930,6 +930,7 @@ DESCRIPTION
930 ext4 930 ext4
931 btrfs 931 btrfs
932 squashfs 932 squashfs
933 erofs
933 swap 934 swap
934 935
935 --fsoptions: Specifies a free-form string of options to be 936 --fsoptions: Specifies a free-form string of options to be
@@ -939,6 +940,12 @@ DESCRIPTION
939 quotes. If not specified, the default string is 940 quotes. If not specified, the default string is
940 "defaults". 941 "defaults".
941 942
943 --fspassno: Specifies the order in which filesystem checks are done
944 at boot time by fsck. See fs_passno parameter of
945 fstab(5). This parameter will be copied into the
946 /etc/fstab file of the installed system. If not
947 specified the default value of "0" will be used.
948
942 --label label: Specifies the label to give to the filesystem 949 --label label: Specifies the label to give to the filesystem
943 to be made on the partition. If the given 950 to be made on the partition. If the given
944 label is already in use by another filesystem, 951 label is already in use by another filesystem,
@@ -990,6 +997,9 @@ DESCRIPTION
990 multiple partitions and we want to keep the right 997 multiple partitions and we want to keep the right
991 permissions and usernames in all the partitions. 998 permissions and usernames in all the partitions.
992 999
1000 --no-fstab-update: This option is specific to wic. It does not update the
1001 '/etc/fstab' stock file for the given partition.
1002
993 --extra-space: This option is specific to wic. It adds extra 1003 --extra-space: This option is specific to wic. It adds extra
994 space after the space filled by the content 1004 space after the space filled by the content
995 of the partition. The final size can go 1005 of the partition. The final size can go
@@ -1108,7 +1118,7 @@ COMMAND:
1108TOPIC: 1118TOPIC:
1109 overview - Presents an overall overview of Wic 1119 overview - Presents an overall overview of Wic
1110 plugins - Presents an overview and API for Wic plugins 1120 plugins - Presents an overview and API for Wic plugins
1111 kickstart - Presents a Wic kicstart file reference 1121 kickstart - Presents a Wic kickstart file reference
1112 1122
1113 1123
1114Examples: 1124Examples:
diff --git a/scripts/lib/wic/ksparser.py b/scripts/lib/wic/ksparser.py
index 3eb669da39..7ef3dc83dd 100644
--- a/scripts/lib/wic/ksparser.py
+++ b/scripts/lib/wic/ksparser.py
@@ -155,9 +155,11 @@ class KickStart():
155 part.add_argument('--change-directory') 155 part.add_argument('--change-directory')
156 part.add_argument("--extra-space", type=sizetype("M")) 156 part.add_argument("--extra-space", type=sizetype("M"))
157 part.add_argument('--fsoptions', dest='fsopts') 157 part.add_argument('--fsoptions', dest='fsopts')
158 part.add_argument('--fspassno', dest='fspassno')
158 part.add_argument('--fstype', default='vfat', 159 part.add_argument('--fstype', default='vfat',
159 choices=('ext2', 'ext3', 'ext4', 'btrfs', 160 choices=('ext2', 'ext3', 'ext4', 'btrfs',
160 'squashfs', 'vfat', 'msdos', 'swap')) 161 'squashfs', 'vfat', 'msdos', 'erofs',
162 'swap', 'none'))
161 part.add_argument('--mkfs-extraopts', default='') 163 part.add_argument('--mkfs-extraopts', default='')
162 part.add_argument('--label') 164 part.add_argument('--label')
163 part.add_argument('--use-label', action='store_true') 165 part.add_argument('--use-label', action='store_true')
@@ -169,6 +171,7 @@ class KickStart():
169 part.add_argument('--rootfs-dir') 171 part.add_argument('--rootfs-dir')
170 part.add_argument('--type', default='primary', 172 part.add_argument('--type', default='primary',
171 choices = ('primary', 'logical')) 173 choices = ('primary', 'logical'))
174 part.add_argument('--hidden', action='store_true')
172 175
173 # --size and --fixed-size cannot be specified together; options 176 # --size and --fixed-size cannot be specified together; options
174 # ----extra-space and --overhead-factor should also raise a parser 177 # ----extra-space and --overhead-factor should also raise a parser
@@ -184,11 +187,13 @@ class KickStart():
184 part.add_argument('--use-uuid', action='store_true') 187 part.add_argument('--use-uuid', action='store_true')
185 part.add_argument('--uuid') 188 part.add_argument('--uuid')
186 part.add_argument('--fsuuid') 189 part.add_argument('--fsuuid')
190 part.add_argument('--no-fstab-update', action='store_true')
191 part.add_argument('--mbr', action='store_true')
187 192
188 bootloader = subparsers.add_parser('bootloader') 193 bootloader = subparsers.add_parser('bootloader')
189 bootloader.add_argument('--append') 194 bootloader.add_argument('--append')
190 bootloader.add_argument('--configfile') 195 bootloader.add_argument('--configfile')
191 bootloader.add_argument('--ptable', choices=('msdos', 'gpt'), 196 bootloader.add_argument('--ptable', choices=('msdos', 'gpt', 'gpt-hybrid'),
192 default='msdos') 197 default='msdos')
193 bootloader.add_argument('--timeout', type=int) 198 bootloader.add_argument('--timeout', type=int)
194 bootloader.add_argument('--source') 199 bootloader.add_argument('--source')
@@ -229,6 +234,10 @@ class KickStart():
229 err = "%s:%d: SquashFS does not support LABEL" \ 234 err = "%s:%d: SquashFS does not support LABEL" \
230 % (confpath, lineno) 235 % (confpath, lineno)
231 raise KickStartError(err) 236 raise KickStartError(err)
237 # erofs does not support filesystem labels
238 if parsed.fstype == 'erofs' and parsed.label:
239 err = "%s:%d: erofs does not support LABEL" % (confpath, lineno)
240 raise KickStartError(err)
232 if parsed.fstype == 'msdos' or parsed.fstype == 'vfat': 241 if parsed.fstype == 'msdos' or parsed.fstype == 'vfat':
233 if parsed.fsuuid: 242 if parsed.fsuuid:
234 if parsed.fsuuid.upper().startswith('0X'): 243 if parsed.fsuuid.upper().startswith('0X'):
diff --git a/scripts/lib/wic/misc.py b/scripts/lib/wic/misc.py
index 57c042c503..1a7c140fa6 100644
--- a/scripts/lib/wic/misc.py
+++ b/scripts/lib/wic/misc.py
@@ -16,16 +16,16 @@ import logging
16import os 16import os
17import re 17import re
18import subprocess 18import subprocess
19import shutil
19 20
20from collections import defaultdict 21from collections import defaultdict
21from distutils import spawn
22 22
23from wic import WicError 23from wic import WicError
24 24
25logger = logging.getLogger('wic') 25logger = logging.getLogger('wic')
26 26
27# executable -> recipe pairs for exec_native_cmd 27# executable -> recipe pairs for exec_native_cmd
28NATIVE_RECIPES = {"bmaptool": "bmap-tools", 28NATIVE_RECIPES = {"bmaptool": "bmaptool",
29 "dumpe2fs": "e2fsprogs", 29 "dumpe2fs": "e2fsprogs",
30 "grub-mkimage": "grub-efi", 30 "grub-mkimage": "grub-efi",
31 "isohybrid": "syslinux", 31 "isohybrid": "syslinux",
@@ -36,6 +36,7 @@ NATIVE_RECIPES = {"bmaptool": "bmap-tools",
36 "mkdosfs": "dosfstools", 36 "mkdosfs": "dosfstools",
37 "mkisofs": "cdrtools", 37 "mkisofs": "cdrtools",
38 "mkfs.btrfs": "btrfs-tools", 38 "mkfs.btrfs": "btrfs-tools",
39 "mkfs.erofs": "erofs-utils",
39 "mkfs.ext2": "e2fsprogs", 40 "mkfs.ext2": "e2fsprogs",
40 "mkfs.ext3": "e2fsprogs", 41 "mkfs.ext3": "e2fsprogs",
41 "mkfs.ext4": "e2fsprogs", 42 "mkfs.ext4": "e2fsprogs",
@@ -122,7 +123,7 @@ def find_executable(cmd, paths):
122 if provided and "%s-native" % recipe in provided: 123 if provided and "%s-native" % recipe in provided:
123 return True 124 return True
124 125
125 return spawn.find_executable(cmd, paths) 126 return shutil.which(cmd, path=paths)
126 127
127def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""): 128def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""):
128 """ 129 """
@@ -140,11 +141,12 @@ def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""):
140 cmd_and_args = pseudo + cmd_and_args 141 cmd_and_args = pseudo + cmd_and_args
141 142
142 hosttools_dir = get_bitbake_var("HOSTTOOLS_DIR") 143 hosttools_dir = get_bitbake_var("HOSTTOOLS_DIR")
144 target_sys = get_bitbake_var("TARGET_SYS")
143 145
144 native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/bin:%s" % \ 146 native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/usr/bin/%s:%s/bin:%s" % \
145 (native_sysroot, native_sysroot, 147 (native_sysroot, native_sysroot,
146 native_sysroot, native_sysroot, 148 native_sysroot, native_sysroot, target_sys,
147 hosttools_dir) 149 native_sysroot, hosttools_dir)
148 150
149 native_cmd_and_args = "export PATH=%s:$PATH;%s" % \ 151 native_cmd_and_args = "export PATH=%s:$PATH;%s" % \
150 (native_paths, cmd_and_args) 152 (native_paths, cmd_and_args)
diff --git a/scripts/lib/wic/partition.py b/scripts/lib/wic/partition.py
index 76d144d12d..795707ec5d 100644
--- a/scripts/lib/wic/partition.py
+++ b/scripts/lib/wic/partition.py
@@ -33,6 +33,7 @@ class Partition():
33 self.include_path = args.include_path 33 self.include_path = args.include_path
34 self.change_directory = args.change_directory 34 self.change_directory = args.change_directory
35 self.fsopts = args.fsopts 35 self.fsopts = args.fsopts
36 self.fspassno = args.fspassno
36 self.fstype = args.fstype 37 self.fstype = args.fstype
37 self.label = args.label 38 self.label = args.label
38 self.use_label = args.use_label 39 self.use_label = args.use_label
@@ -54,9 +55,12 @@ class Partition():
54 self.uuid = args.uuid 55 self.uuid = args.uuid
55 self.fsuuid = args.fsuuid 56 self.fsuuid = args.fsuuid
56 self.type = args.type 57 self.type = args.type
58 self.no_fstab_update = args.no_fstab_update
57 self.updated_fstab_path = None 59 self.updated_fstab_path = None
58 self.has_fstab = False 60 self.has_fstab = False
59 self.update_fstab_in_rootfs = False 61 self.update_fstab_in_rootfs = False
62 self.hidden = args.hidden
63 self.mbr = args.mbr
60 64
61 self.lineno = lineno 65 self.lineno = lineno
62 self.source_file = "" 66 self.source_file = ""
@@ -104,7 +108,7 @@ class Partition():
104 extra_blocks = self.extra_space 108 extra_blocks = self.extra_space
105 109
106 rootfs_size = actual_rootfs_size + extra_blocks 110 rootfs_size = actual_rootfs_size + extra_blocks
107 rootfs_size *= self.overhead_factor 111 rootfs_size = int(rootfs_size * self.overhead_factor)
108 112
109 logger.debug("Added %d extra blocks to %s to get to %d total blocks", 113 logger.debug("Added %d extra blocks to %s to get to %d total blocks",
110 extra_blocks, self.mountpoint, rootfs_size) 114 extra_blocks, self.mountpoint, rootfs_size)
@@ -131,6 +135,8 @@ class Partition():
131 self.update_fstab_in_rootfs = True 135 self.update_fstab_in_rootfs = True
132 136
133 if not self.source: 137 if not self.source:
138 if self.fstype == "none" or self.no_table:
139 return
134 if not self.size and not self.fixed_size: 140 if not self.size and not self.fixed_size:
135 raise WicError("The %s partition has a size of zero. Please " 141 raise WicError("The %s partition has a size of zero. Please "
136 "specify a non-zero --size/--fixed-size for that " 142 "specify a non-zero --size/--fixed-size for that "
@@ -141,9 +147,9 @@ class Partition():
141 native_sysroot) 147 native_sysroot)
142 self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype) 148 self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype)
143 else: 149 else:
144 if self.fstype == 'squashfs': 150 if self.fstype in ('squashfs', 'erofs'):
145 raise WicError("It's not possible to create empty squashfs " 151 raise WicError("It's not possible to create empty %s "
146 "partition '%s'" % (self.mountpoint)) 152 "partition '%s'" % (self.fstype, self.mountpoint))
147 153
148 rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label, 154 rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label,
149 self.lineno, self.fstype) 155 self.lineno, self.fstype)
@@ -170,7 +176,7 @@ class Partition():
170 # Split sourceparams string of the form key1=val1[,key2=val2,...] 176 # Split sourceparams string of the form key1=val1[,key2=val2,...]
171 # into a dict. Also accepts valueless keys i.e. without = 177 # into a dict. Also accepts valueless keys i.e. without =
172 splitted = self.sourceparams.split(',') 178 splitted = self.sourceparams.split(',')
173 srcparams_dict = dict(par.split('=', 1) for par in splitted if par) 179 srcparams_dict = dict((par.split('=', 1) + [None])[:2] for par in splitted if par)
174 180
175 plugin = PluginMgr.get_plugins('source')[self.source] 181 plugin = PluginMgr.get_plugins('source')[self.source]
176 plugin.do_configure_partition(self, srcparams_dict, creator, 182 plugin.do_configure_partition(self, srcparams_dict, creator,
@@ -278,6 +284,20 @@ class Partition():
278 284
279 extraopts = self.mkfs_extraopts or "-F -i 8192" 285 extraopts = self.mkfs_extraopts or "-F -i 8192"
280 286
287 if os.getenv('SOURCE_DATE_EPOCH'):
288 sde_time = int(os.getenv('SOURCE_DATE_EPOCH'))
289 if pseudo:
290 pseudo = "export E2FSPROGS_FAKE_TIME=%s;%s " % (sde_time, pseudo)
291 else:
292 pseudo = "export E2FSPROGS_FAKE_TIME=%s; " % sde_time
293
294 # Set hash_seed to generate deterministic directory indexes
295 namespace = uuid.UUID("e7429877-e7b3-4a68-a5c9-2f2fdf33d460")
296 if self.fsuuid:
297 namespace = uuid.UUID(self.fsuuid)
298 hash_seed = str(uuid.uuid5(namespace, str(sde_time)))
299 extraopts += " -E hash_seed=%s" % hash_seed
300
281 label_str = "" 301 label_str = ""
282 if self.label: 302 if self.label:
283 label_str = "-L %s" % self.label 303 label_str = "-L %s" % self.label
@@ -286,7 +306,7 @@ class Partition():
286 (self.fstype, extraopts, rootfs, label_str, self.fsuuid, rootfs_dir) 306 (self.fstype, extraopts, rootfs, label_str, self.fsuuid, rootfs_dir)
287 exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) 307 exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
288 308
289 if self.updated_fstab_path and self.has_fstab: 309 if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
290 debugfs_script_path = os.path.join(cr_workdir, "debugfs_script") 310 debugfs_script_path = os.path.join(cr_workdir, "debugfs_script")
291 with open(debugfs_script_path, "w") as f: 311 with open(debugfs_script_path, "w") as f:
292 f.write("cd etc\n") 312 f.write("cd etc\n")
@@ -298,6 +318,30 @@ class Partition():
298 mkfs_cmd = "fsck.%s -pvfD %s" % (self.fstype, rootfs) 318 mkfs_cmd = "fsck.%s -pvfD %s" % (self.fstype, rootfs)
299 exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo) 319 exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
300 320
321 if os.getenv('SOURCE_DATE_EPOCH'):
322 sde_time = hex(int(os.getenv('SOURCE_DATE_EPOCH')))
323 debugfs_script_path = os.path.join(cr_workdir, "debugfs_script")
324 files = []
325 for root, dirs, others in os.walk(rootfs_dir):
326 base = root.replace(rootfs_dir, "").rstrip(os.sep)
327 files += [ "/" if base == "" else base ]
328 files += [ base + "/" + n for n in dirs + others ]
329 with open(debugfs_script_path, "w") as f:
330 f.write("set_current_time %s\n" % (sde_time))
331 if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
332 f.write("set_inode_field /etc/fstab mtime %s\n" % (sde_time))
333 f.write("set_inode_field /etc/fstab mtime_extra 0\n")
334 for file in set(files):
335 for time in ["atime", "ctime", "crtime"]:
336 f.write("set_inode_field \"%s\" %s %s\n" % (file, time, sde_time))
337 f.write("set_inode_field \"%s\" %s_extra 0\n" % (file, time))
338 for time in ["wtime", "mkfs_time", "lastcheck"]:
339 f.write("set_super_value %s %s\n" % (time, sde_time))
340 for time in ["mtime", "first_error_time", "last_error_time"]:
341 f.write("set_super_value %s 0\n" % (time))
342 debugfs_cmd = "debugfs -w -f %s %s" % (debugfs_script_path, rootfs)
343 exec_native_cmd(debugfs_cmd, native_sysroot)
344
301 self.check_for_Y2038_problem(rootfs, native_sysroot) 345 self.check_for_Y2038_problem(rootfs, native_sysroot)
302 346
303 def prepare_rootfs_btrfs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir, 347 def prepare_rootfs_btrfs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
@@ -350,8 +394,8 @@ class Partition():
350 mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir) 394 mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir)
351 exec_native_cmd(mcopy_cmd, native_sysroot) 395 exec_native_cmd(mcopy_cmd, native_sysroot)
352 396
353 if self.updated_fstab_path and self.has_fstab: 397 if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
354 mcopy_cmd = "mcopy -i %s %s ::/etc/fstab" % (rootfs, self.updated_fstab_path) 398 mcopy_cmd = "mcopy -m -i %s %s ::/etc/fstab" % (rootfs, self.updated_fstab_path)
355 exec_native_cmd(mcopy_cmd, native_sysroot) 399 exec_native_cmd(mcopy_cmd, native_sysroot)
356 400
357 chmod_cmd = "chmod 644 %s" % rootfs 401 chmod_cmd = "chmod 644 %s" % rootfs
@@ -369,6 +413,19 @@ class Partition():
369 (rootfs_dir, rootfs, extraopts) 413 (rootfs_dir, rootfs, extraopts)
370 exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo) 414 exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo)
371 415
416 def prepare_rootfs_erofs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
417 native_sysroot, pseudo):
418 """
419 Prepare content for a erofs rootfs partition.
420 """
421 extraopts = self.mkfs_extraopts or ''
422 erofs_cmd = "mkfs.erofs %s -U %s %s %s" % \
423 (extraopts, self.fsuuid, rootfs, rootfs_dir)
424 exec_native_cmd(erofs_cmd, native_sysroot, pseudo=pseudo)
425
426 def prepare_empty_partition_none(self, rootfs, oe_builddir, native_sysroot):
427 pass
428
372 def prepare_empty_partition_ext(self, rootfs, oe_builddir, 429 def prepare_empty_partition_ext(self, rootfs, oe_builddir,
373 native_sysroot): 430 native_sysroot):
374 """ 431 """
diff --git a/scripts/lib/wic/pluginbase.py b/scripts/lib/wic/pluginbase.py
index d9b4e57747..b64568339b 100644
--- a/scripts/lib/wic/pluginbase.py
+++ b/scripts/lib/wic/pluginbase.py
@@ -9,9 +9,11 @@ __all__ = ['ImagerPlugin', 'SourcePlugin']
9 9
10import os 10import os
11import logging 11import logging
12import types
12 13
13from collections import defaultdict 14from collections import defaultdict
14from importlib.machinery import SourceFileLoader 15import importlib
16import importlib.util
15 17
16from wic import WicError 18from wic import WicError
17from wic.misc import get_bitbake_var 19from wic.misc import get_bitbake_var
@@ -54,7 +56,9 @@ class PluginMgr:
54 mname = fname[:-3] 56 mname = fname[:-3]
55 mpath = os.path.join(ppath, fname) 57 mpath = os.path.join(ppath, fname)
56 logger.debug("loading plugin module %s", mpath) 58 logger.debug("loading plugin module %s", mpath)
57 SourceFileLoader(mname, mpath).load_module() 59 spec = importlib.util.spec_from_file_location(mname, mpath)
60 module = importlib.util.module_from_spec(spec)
61 spec.loader.exec_module(module)
58 62
59 return PLUGINS.get(ptype) 63 return PLUGINS.get(ptype)
60 64
diff --git a/scripts/lib/wic/plugins/imager/direct.py b/scripts/lib/wic/plugins/imager/direct.py
index ea709e8c54..a1d152659b 100644
--- a/scripts/lib/wic/plugins/imager/direct.py
+++ b/scripts/lib/wic/plugins/imager/direct.py
@@ -77,7 +77,8 @@ class DirectPlugin(ImagerPlugin):
77 77
78 image_path = self._full_path(self.workdir, self.parts[0].disk, "direct") 78 image_path = self._full_path(self.workdir, self.parts[0].disk, "direct")
79 self._image = PartitionedImage(image_path, self.ptable_format, 79 self._image = PartitionedImage(image_path, self.ptable_format,
80 self.parts, self.native_sysroot) 80 self.parts, self.native_sysroot,
81 options.extra_space)
81 82
82 def setup_workdir(self, workdir): 83 def setup_workdir(self, workdir):
83 if workdir: 84 if workdir:
@@ -116,7 +117,7 @@ class DirectPlugin(ImagerPlugin):
116 updated = False 117 updated = False
117 for part in self.parts: 118 for part in self.parts:
118 if not part.realnum or not part.mountpoint \ 119 if not part.realnum or not part.mountpoint \
119 or part.mountpoint == "/": 120 or part.mountpoint == "/" or not (part.mountpoint.startswith('/') or part.mountpoint == "swap"):
120 continue 121 continue
121 122
122 if part.use_uuid: 123 if part.use_uuid:
@@ -137,8 +138,9 @@ class DirectPlugin(ImagerPlugin):
137 device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum) 138 device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum)
138 139
139 opts = part.fsopts if part.fsopts else "defaults" 140 opts = part.fsopts if part.fsopts else "defaults"
141 passno = part.fspassno if part.fspassno else "0"
140 line = "\t".join([device_name, part.mountpoint, part.fstype, 142 line = "\t".join([device_name, part.mountpoint, part.fstype,
141 opts, "0", "0"]) + "\n" 143 opts, "0", passno]) + "\n"
142 144
143 fstab_lines.append(line) 145 fstab_lines.append(line)
144 updated = True 146 updated = True
@@ -147,6 +149,9 @@ class DirectPlugin(ImagerPlugin):
147 self.updated_fstab_path = os.path.join(self.workdir, "fstab") 149 self.updated_fstab_path = os.path.join(self.workdir, "fstab")
148 with open(self.updated_fstab_path, "w") as f: 150 with open(self.updated_fstab_path, "w") as f:
149 f.writelines(fstab_lines) 151 f.writelines(fstab_lines)
152 if os.getenv('SOURCE_DATE_EPOCH'):
153 fstab_time = int(os.getenv('SOURCE_DATE_EPOCH'))
154 os.utime(self.updated_fstab_path, (fstab_time, fstab_time))
150 155
151 def _full_path(self, path, name, extention): 156 def _full_path(self, path, name, extention):
152 """ Construct full file path to a file we generate. """ 157 """ Construct full file path to a file we generate. """
@@ -258,6 +263,8 @@ class DirectPlugin(ImagerPlugin):
258 if part.mountpoint == "/": 263 if part.mountpoint == "/":
259 if part.uuid: 264 if part.uuid:
260 return "PARTUUID=%s" % part.uuid 265 return "PARTUUID=%s" % part.uuid
266 elif part.label and self.ptable_format != 'msdos':
267 return "PARTLABEL=%s" % part.label
261 else: 268 else:
262 suffix = 'p' if part.disk.startswith('mmcblk') else '' 269 suffix = 'p' if part.disk.startswith('mmcblk') else ''
263 return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum) 270 return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum)
@@ -293,7 +300,7 @@ class PartitionedImage():
293 Partitioned image in a file. 300 Partitioned image in a file.
294 """ 301 """
295 302
296 def __init__(self, path, ptable_format, partitions, native_sysroot=None): 303 def __init__(self, path, ptable_format, partitions, native_sysroot=None, extra_space=0):
297 self.path = path # Path to the image file 304 self.path = path # Path to the image file
298 self.numpart = 0 # Number of allocated partitions 305 self.numpart = 0 # Number of allocated partitions
299 self.realpart = 0 # Number of partitions in the partition table 306 self.realpart = 0 # Number of partitions in the partition table
@@ -306,7 +313,10 @@ class PartitionedImage():
306 # all partitions (in bytes) 313 # all partitions (in bytes)
307 self.ptable_format = ptable_format # Partition table format 314 self.ptable_format = ptable_format # Partition table format
308 # Disk system identifier 315 # Disk system identifier
309 self.identifier = random.SystemRandom().randint(1, 0xffffffff) 316 if os.getenv('SOURCE_DATE_EPOCH'):
317 self.identifier = random.Random(int(os.getenv('SOURCE_DATE_EPOCH'))).randint(1, 0xffffffff)
318 else:
319 self.identifier = random.SystemRandom().randint(1, 0xffffffff)
310 320
311 self.partitions = partitions 321 self.partitions = partitions
312 self.partimages = [] 322 self.partimages = []
@@ -314,6 +324,7 @@ class PartitionedImage():
314 self.sector_size = SECTOR_SIZE 324 self.sector_size = SECTOR_SIZE
315 self.native_sysroot = native_sysroot 325 self.native_sysroot = native_sysroot
316 num_real_partitions = len([p for p in self.partitions if not p.no_table]) 326 num_real_partitions = len([p for p in self.partitions if not p.no_table])
327 self.extra_space = extra_space
317 328
318 # calculate the real partition number, accounting for partitions not 329 # calculate the real partition number, accounting for partitions not
319 # in the partition table and logical partitions 330 # in the partition table and logical partitions
@@ -331,7 +342,7 @@ class PartitionedImage():
331 # generate parition and filesystem UUIDs 342 # generate parition and filesystem UUIDs
332 for part in self.partitions: 343 for part in self.partitions:
333 if not part.uuid and part.use_uuid: 344 if not part.uuid and part.use_uuid:
334 if self.ptable_format == 'gpt': 345 if self.ptable_format in ('gpt', 'gpt-hybrid'):
335 part.uuid = str(uuid.uuid4()) 346 part.uuid = str(uuid.uuid4())
336 else: # msdos partition table 347 else: # msdos partition table
337 part.uuid = '%08x-%02d' % (self.identifier, part.realnum) 348 part.uuid = '%08x-%02d' % (self.identifier, part.realnum)
@@ -387,6 +398,10 @@ class PartitionedImage():
387 raise WicError("setting custom partition type is not " \ 398 raise WicError("setting custom partition type is not " \
388 "implemented for msdos partitions") 399 "implemented for msdos partitions")
389 400
401 if part.mbr and self.ptable_format != 'gpt-hybrid':
402 raise WicError("Partition may only be included in MBR with " \
403 "a gpt-hybrid partition table")
404
390 # Get the disk where the partition is located 405 # Get the disk where the partition is located
391 self.numpart += 1 406 self.numpart += 1
392 if not part.no_table: 407 if not part.no_table:
@@ -395,7 +410,7 @@ class PartitionedImage():
395 if self.numpart == 1: 410 if self.numpart == 1:
396 if self.ptable_format == "msdos": 411 if self.ptable_format == "msdos":
397 overhead = MBR_OVERHEAD 412 overhead = MBR_OVERHEAD
398 elif self.ptable_format == "gpt": 413 elif self.ptable_format in ("gpt", "gpt-hybrid"):
399 overhead = GPT_OVERHEAD 414 overhead = GPT_OVERHEAD
400 415
401 # Skip one sector required for the partitioning scheme overhead 416 # Skip one sector required for the partitioning scheme overhead
@@ -479,10 +494,11 @@ class PartitionedImage():
479 # Once all the partitions have been layed out, we can calculate the 494 # Once all the partitions have been layed out, we can calculate the
480 # minumim disk size 495 # minumim disk size
481 self.min_size = self.offset 496 self.min_size = self.offset
482 if self.ptable_format == "gpt": 497 if self.ptable_format in ("gpt", "gpt-hybrid"):
483 self.min_size += GPT_OVERHEAD 498 self.min_size += GPT_OVERHEAD
484 499
485 self.min_size *= self.sector_size 500 self.min_size *= self.sector_size
501 self.min_size += self.extra_space
486 502
487 def _create_partition(self, device, parttype, fstype, start, size): 503 def _create_partition(self, device, parttype, fstype, start, size):
488 """ Create a partition on an image described by the 'device' object. """ 504 """ Create a partition on an image described by the 'device' object. """
@@ -499,22 +515,49 @@ class PartitionedImage():
499 515
500 return exec_native_cmd(cmd, self.native_sysroot) 516 return exec_native_cmd(cmd, self.native_sysroot)
501 517
518 def _write_identifier(self, device, identifier):
519 logger.debug("Set disk identifier %x", identifier)
520 with open(device, 'r+b') as img:
521 img.seek(0x1B8)
522 img.write(identifier.to_bytes(4, 'little'))
523
524 def _make_disk(self, device, ptable_format, min_size):
525 logger.debug("Creating sparse file %s", device)
526 with open(device, 'w') as sparse:
527 os.ftruncate(sparse.fileno(), min_size)
528
529 logger.debug("Initializing partition table for %s", device)
530 exec_native_cmd("parted -s %s mklabel %s" % (device, ptable_format),
531 self.native_sysroot)
532
533 def _write_disk_guid(self):
534 if self.ptable_format in ('gpt', 'gpt-hybrid'):
535 if os.getenv('SOURCE_DATE_EPOCH'):
536 self.disk_guid = uuid.UUID(int=int(os.getenv('SOURCE_DATE_EPOCH')))
537 else:
538 self.disk_guid = uuid.uuid4()
539
540 logger.debug("Set disk guid %s", self.disk_guid)
541 sfdisk_cmd = "sfdisk --disk-id %s %s" % (self.path, self.disk_guid)
542 exec_native_cmd(sfdisk_cmd, self.native_sysroot)
543
502 def create(self): 544 def create(self):
503 logger.debug("Creating sparse file %s", self.path) 545 self._make_disk(self.path,
504 with open(self.path, 'w') as sparse: 546 "gpt" if self.ptable_format == "gpt-hybrid" else self.ptable_format,
505 os.ftruncate(sparse.fileno(), self.min_size) 547 self.min_size)
506 548
507 logger.debug("Initializing partition table for %s", self.path) 549 self._write_identifier(self.path, self.identifier)
508 exec_native_cmd("parted -s %s mklabel %s" % 550 self._write_disk_guid()
509 (self.path, self.ptable_format), self.native_sysroot)
510 551
511 logger.debug("Set disk identifier %x", self.identifier) 552 if self.ptable_format == "gpt-hybrid":
512 with open(self.path, 'r+b') as img: 553 mbr_path = self.path + ".mbr"
513 img.seek(0x1B8) 554 self._make_disk(mbr_path, "msdos", self.min_size)
514 img.write(self.identifier.to_bytes(4, 'little')) 555 self._write_identifier(mbr_path, self.identifier)
515 556
516 logger.debug("Creating partitions") 557 logger.debug("Creating partitions")
517 558
559 hybrid_mbr_part_num = 0
560
518 for part in self.partitions: 561 for part in self.partitions:
519 if part.num == 0: 562 if part.num == 0:
520 continue 563 continue
@@ -559,11 +602,19 @@ class PartitionedImage():
559 self._create_partition(self.path, part.type, 602 self._create_partition(self.path, part.type,
560 parted_fs_type, part.start, part.size_sec) 603 parted_fs_type, part.start, part.size_sec)
561 604
562 if part.part_name: 605 if self.ptable_format == "gpt-hybrid" and part.mbr:
606 hybrid_mbr_part_num += 1
607 if hybrid_mbr_part_num > 4:
608 raise WicError("Extended MBR partitions are not supported in hybrid MBR")
609 self._create_partition(mbr_path, "primary",
610 parted_fs_type, part.start, part.size_sec)
611
612 if self.ptable_format in ("gpt", "gpt-hybrid") and (part.part_name or part.label):
613 partition_label = part.part_name if part.part_name else part.label
563 logger.debug("partition %d: set name to %s", 614 logger.debug("partition %d: set name to %s",
564 part.num, part.part_name) 615 part.num, partition_label)
565 exec_native_cmd("sgdisk --change-name=%d:%s %s" % \ 616 exec_native_cmd("sgdisk --change-name=%d:%s %s" % \
566 (part.num, part.part_name, 617 (part.num, partition_label,
567 self.path), self.native_sysroot) 618 self.path), self.native_sysroot)
568 619
569 if part.part_type: 620 if part.part_type:
@@ -573,32 +624,55 @@ class PartitionedImage():
573 (part.num, part.part_type, 624 (part.num, part.part_type,
574 self.path), self.native_sysroot) 625 self.path), self.native_sysroot)
575 626
576 if part.uuid and self.ptable_format == "gpt": 627 if part.uuid and self.ptable_format in ("gpt", "gpt-hybrid"):
577 logger.debug("partition %d: set UUID to %s", 628 logger.debug("partition %d: set UUID to %s",
578 part.num, part.uuid) 629 part.num, part.uuid)
579 exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \ 630 exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \
580 (part.num, part.uuid, self.path), 631 (part.num, part.uuid, self.path),
581 self.native_sysroot) 632 self.native_sysroot)
582 633
583 if part.label and self.ptable_format == "gpt":
584 logger.debug("partition %d: set name to %s",
585 part.num, part.label)
586 exec_native_cmd("parted -s %s name %d %s" % \
587 (self.path, part.num, part.label),
588 self.native_sysroot)
589
590 if part.active: 634 if part.active:
591 flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot" 635 flag_name = "legacy_boot" if self.ptable_format in ('gpt', 'gpt-hybrid') else "boot"
592 logger.debug("Set '%s' flag for partition '%s' on disk '%s'", 636 logger.debug("Set '%s' flag for partition '%s' on disk '%s'",
593 flag_name, part.num, self.path) 637 flag_name, part.num, self.path)
594 exec_native_cmd("parted -s %s set %d %s on" % \ 638 exec_native_cmd("parted -s %s set %d %s on" % \
595 (self.path, part.num, flag_name), 639 (self.path, part.num, flag_name),
596 self.native_sysroot) 640 self.native_sysroot)
641 if self.ptable_format == 'gpt-hybrid' and part.mbr:
642 exec_native_cmd("parted -s %s set %d %s on" % \
643 (mbr_path, hybrid_mbr_part_num, "boot"),
644 self.native_sysroot)
597 if part.system_id: 645 if part.system_id:
598 exec_native_cmd("sfdisk --part-type %s %s %s" % \ 646 exec_native_cmd("sfdisk --part-type %s %s %s" % \
599 (self.path, part.num, part.system_id), 647 (self.path, part.num, part.system_id),
600 self.native_sysroot) 648 self.native_sysroot)
601 649
650 if part.hidden and self.ptable_format == "gpt":
651 logger.debug("Set hidden attribute for partition '%s' on disk '%s'",
652 part.num, self.path)
653 exec_native_cmd("sfdisk --part-attrs %s %s RequiredPartition" % \
654 (self.path, part.num),
655 self.native_sysroot)
656
657 if self.ptable_format == "gpt-hybrid":
658 # Write a protective GPT partition
659 hybrid_mbr_part_num += 1
660 if hybrid_mbr_part_num > 4:
661 raise WicError("Extended MBR partitions are not supported in hybrid MBR")
662
663 # parted cannot directly create a protective GPT partition, so
664 # create with an arbitrary type, then change it to the correct type
665 # with sfdisk
666 self._create_partition(mbr_path, "primary", "fat32", 1, GPT_OVERHEAD)
667 exec_native_cmd("sfdisk --part-type %s %d 0xee" % (mbr_path, hybrid_mbr_part_num),
668 self.native_sysroot)
669
670 # Copy hybrid MBR
671 with open(mbr_path, "rb") as mbr_file:
672 with open(self.path, "r+b") as image_file:
673 mbr = mbr_file.read(512)
674 image_file.write(mbr)
675
602 def cleanup(self): 676 def cleanup(self):
603 pass 677 pass
604 678
diff --git a/scripts/lib/wic/plugins/source/bootimg-efi.py b/scripts/lib/wic/plugins/source/bootimg-efi.py
index cdc72543c2..13a9cddf4e 100644
--- a/scripts/lib/wic/plugins/source/bootimg-efi.py
+++ b/scripts/lib/wic/plugins/source/bootimg-efi.py
@@ -12,6 +12,7 @@
12 12
13import logging 13import logging
14import os 14import os
15import tempfile
15import shutil 16import shutil
16import re 17import re
17 18
@@ -34,6 +35,26 @@ class BootimgEFIPlugin(SourcePlugin):
34 name = 'bootimg-efi' 35 name = 'bootimg-efi'
35 36
36 @classmethod 37 @classmethod
38 def _copy_additional_files(cls, hdddir, initrd, dtb):
39 bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
40 if not bootimg_dir:
41 raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
42
43 if initrd:
44 initrds = initrd.split(';')
45 for rd in initrds:
46 cp_cmd = "cp %s/%s %s" % (bootimg_dir, rd, hdddir)
47 exec_cmd(cp_cmd, True)
48 else:
49 logger.debug("Ignoring missing initrd")
50
51 if dtb:
52 if ';' in dtb:
53 raise WicError("Only one DTB supported, exiting")
54 cp_cmd = "cp %s/%s %s" % (bootimg_dir, dtb, hdddir)
55 exec_cmd(cp_cmd, True)
56
57 @classmethod
37 def do_configure_grubefi(cls, hdddir, creator, cr_workdir, source_params): 58 def do_configure_grubefi(cls, hdddir, creator, cr_workdir, source_params):
38 """ 59 """
39 Create loader-specific (grub-efi) config 60 Create loader-specific (grub-efi) config
@@ -52,18 +73,9 @@ class BootimgEFIPlugin(SourcePlugin):
52 "get it from %s." % configfile) 73 "get it from %s." % configfile)
53 74
54 initrd = source_params.get('initrd') 75 initrd = source_params.get('initrd')
76 dtb = source_params.get('dtb')
55 77
56 if initrd: 78 cls._copy_additional_files(hdddir, initrd, dtb)
57 bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
58 if not bootimg_dir:
59 raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
60
61 initrds = initrd.split(';')
62 for rd in initrds:
63 cp_cmd = "cp %s/%s %s" % (bootimg_dir, rd, hdddir)
64 exec_cmd(cp_cmd, True)
65 else:
66 logger.debug("Ignoring missing initrd")
67 79
68 if not custom_cfg: 80 if not custom_cfg:
69 # Create grub configuration using parameters from wks file 81 # Create grub configuration using parameters from wks file
@@ -97,6 +109,9 @@ class BootimgEFIPlugin(SourcePlugin):
97 grubefi_conf += " /%s" % rd 109 grubefi_conf += " /%s" % rd
98 grubefi_conf += "\n" 110 grubefi_conf += "\n"
99 111
112 if dtb:
113 grubefi_conf += "devicetree /%s\n" % dtb
114
100 grubefi_conf += "}\n" 115 grubefi_conf += "}\n"
101 116
102 logger.debug("Writing grubefi config %s/hdd/boot/EFI/BOOT/grub.cfg", 117 logger.debug("Writing grubefi config %s/hdd/boot/EFI/BOOT/grub.cfg",
@@ -118,24 +133,18 @@ class BootimgEFIPlugin(SourcePlugin):
118 133
119 bootloader = creator.ks.bootloader 134 bootloader = creator.ks.bootloader
120 135
136 unified_image = source_params.get('create-unified-kernel-image') == "true"
137
121 loader_conf = "" 138 loader_conf = ""
122 loader_conf += "default boot\n" 139 if not unified_image:
140 loader_conf += "default boot\n"
123 loader_conf += "timeout %d\n" % bootloader.timeout 141 loader_conf += "timeout %d\n" % bootloader.timeout
124 142
125 initrd = source_params.get('initrd') 143 initrd = source_params.get('initrd')
144 dtb = source_params.get('dtb')
126 145
127 if initrd: 146 if not unified_image:
128 # obviously we need to have a common common deploy var 147 cls._copy_additional_files(hdddir, initrd, dtb)
129 bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
130 if not bootimg_dir:
131 raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
132
133 initrds = initrd.split(';')
134 for rd in initrds:
135 cp_cmd = "cp %s/%s %s" % (bootimg_dir, rd, hdddir)
136 exec_cmd(cp_cmd, True)
137 else:
138 logger.debug("Ignoring missing initrd")
139 148
140 logger.debug("Writing systemd-boot config " 149 logger.debug("Writing systemd-boot config "
141 "%s/hdd/boot/loader/loader.conf", cr_workdir) 150 "%s/hdd/boot/loader/loader.conf", cr_workdir)
@@ -183,11 +192,15 @@ class BootimgEFIPlugin(SourcePlugin):
183 for rd in initrds: 192 for rd in initrds:
184 boot_conf += "initrd /%s\n" % rd 193 boot_conf += "initrd /%s\n" % rd
185 194
186 logger.debug("Writing systemd-boot config " 195 if dtb:
187 "%s/hdd/boot/loader/entries/boot.conf", cr_workdir) 196 boot_conf += "devicetree /%s\n" % dtb
188 cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w") 197
189 cfg.write(boot_conf) 198 if not unified_image:
190 cfg.close() 199 logger.debug("Writing systemd-boot config "
200 "%s/hdd/boot/loader/entries/boot.conf", cr_workdir)
201 cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w")
202 cfg.write(boot_conf)
203 cfg.close()
191 204
192 205
193 @classmethod 206 @classmethod
@@ -207,6 +220,8 @@ class BootimgEFIPlugin(SourcePlugin):
207 cls.do_configure_grubefi(hdddir, creator, cr_workdir, source_params) 220 cls.do_configure_grubefi(hdddir, creator, cr_workdir, source_params)
208 elif source_params['loader'] == 'systemd-boot': 221 elif source_params['loader'] == 'systemd-boot':
209 cls.do_configure_systemdboot(hdddir, creator, cr_workdir, source_params) 222 cls.do_configure_systemdboot(hdddir, creator, cr_workdir, source_params)
223 elif source_params['loader'] == 'uefi-kernel':
224 pass
210 else: 225 else:
211 raise WicError("unrecognized bootimg-efi loader: %s" % source_params['loader']) 226 raise WicError("unrecognized bootimg-efi loader: %s" % source_params['loader'])
212 except KeyError: 227 except KeyError:
@@ -288,9 +303,107 @@ class BootimgEFIPlugin(SourcePlugin):
288 kernel = "%s-%s.bin" % \ 303 kernel = "%s-%s.bin" % \
289 (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) 304 (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
290 305
291 install_cmd = "install -m 0644 %s/%s %s/%s" % \ 306 if source_params.get('create-unified-kernel-image') == "true":
292 (staging_kernel_dir, kernel, hdddir, kernel) 307 initrd = source_params.get('initrd')
293 exec_cmd(install_cmd) 308 if not initrd:
309 raise WicError("initrd= must be specified when create-unified-kernel-image=true, exiting")
310
311 deploy_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
312 efi_stub = glob("%s/%s" % (deploy_dir, "linux*.efi.stub"))
313 if len(efi_stub) == 0:
314 raise WicError("Unified Kernel Image EFI stub not found, exiting")
315 efi_stub = efi_stub[0]
316
317 with tempfile.TemporaryDirectory() as tmp_dir:
318 label = source_params.get('label')
319 label_conf = "root=%s" % creator.rootdev
320 if label:
321 label_conf = "LABEL=%s" % label
322
323 bootloader = creator.ks.bootloader
324 cmdline = open("%s/cmdline" % tmp_dir, "w")
325 cmdline.write("%s %s" % (label_conf, bootloader.append))
326 cmdline.close()
327
328 initrds = initrd.split(';')
329 initrd = open("%s/initrd" % tmp_dir, "wb")
330 for f in initrds:
331 with open("%s/%s" % (deploy_dir, f), 'rb') as in_file:
332 shutil.copyfileobj(in_file, initrd)
333 initrd.close()
334
335 # Searched by systemd-boot:
336 # https://systemd.io/BOOT_LOADER_SPECIFICATION/#type-2-efi-unified-kernel-images
337 install_cmd = "install -d %s/EFI/Linux" % hdddir
338 exec_cmd(install_cmd)
339
340 staging_dir_host = get_bitbake_var("STAGING_DIR_HOST")
341 target_sys = get_bitbake_var("TARGET_SYS")
342
343 objdump_cmd = "%s-objdump" % target_sys
344 objdump_cmd += " -p %s" % efi_stub
345 objdump_cmd += " | awk '{ if ($1 == \"SectionAlignment\"){print $2} }'"
346
347 ret, align_str = exec_native_cmd(objdump_cmd, native_sysroot)
348 align = int(align_str, 16)
349
350 objdump_cmd = "%s-objdump" % target_sys
351 objdump_cmd += " -h %s | tail -2" % efi_stub
352 ret, output = exec_native_cmd(objdump_cmd, native_sysroot)
353
354 offset = int(output.split()[2], 16) + int(output.split()[3], 16)
355
356 osrel_off = offset + align - offset % align
357 osrel_path = "%s/usr/lib/os-release" % staging_dir_host
358 osrel_sz = os.stat(osrel_path).st_size
359
360 cmdline_off = osrel_off + osrel_sz
361 cmdline_off = cmdline_off + align - cmdline_off % align
362 cmdline_sz = os.stat(cmdline.name).st_size
363
364 dtb_off = cmdline_off + cmdline_sz
365 dtb_off = dtb_off + align - dtb_off % align
366
367 dtb = source_params.get('dtb')
368 if dtb:
369 if ';' in dtb:
370 raise WicError("Only one DTB supported, exiting")
371 dtb_path = "%s/%s" % (deploy_dir, dtb)
372 dtb_params = '--add-section .dtb=%s --change-section-vma .dtb=0x%x' % \
373 (dtb_path, dtb_off)
374 linux_off = dtb_off + os.stat(dtb_path).st_size
375 linux_off = linux_off + align - linux_off % align
376 else:
377 dtb_params = ''
378 linux_off = dtb_off
379
380 linux_path = "%s/%s" % (staging_kernel_dir, kernel)
381 linux_sz = os.stat(linux_path).st_size
382
383 initrd_off = linux_off + linux_sz
384 initrd_off = initrd_off + align - initrd_off % align
385
386 # https://www.freedesktop.org/software/systemd/man/systemd-stub.html
387 objcopy_cmd = "%s-objcopy" % target_sys
388 objcopy_cmd += " --enable-deterministic-archives"
389 objcopy_cmd += " --preserve-dates"
390 objcopy_cmd += " --add-section .osrel=%s" % osrel_path
391 objcopy_cmd += " --change-section-vma .osrel=0x%x" % osrel_off
392 objcopy_cmd += " --add-section .cmdline=%s" % cmdline.name
393 objcopy_cmd += " --change-section-vma .cmdline=0x%x" % cmdline_off
394 objcopy_cmd += dtb_params
395 objcopy_cmd += " --add-section .linux=%s" % linux_path
396 objcopy_cmd += " --change-section-vma .linux=0x%x" % linux_off
397 objcopy_cmd += " --add-section .initrd=%s" % initrd.name
398 objcopy_cmd += " --change-section-vma .initrd=0x%x" % initrd_off
399 objcopy_cmd += " %s %s/EFI/Linux/linux.efi" % (efi_stub, hdddir)
400
401 exec_native_cmd(objcopy_cmd, native_sysroot)
402 else:
403 if source_params.get('install-kernel-into-boot-dir') != 'false':
404 install_cmd = "install -m 0644 %s/%s %s/%s" % \
405 (staging_kernel_dir, kernel, hdddir, kernel)
406 exec_cmd(install_cmd)
294 407
295 if get_bitbake_var("IMAGE_EFI_BOOT_FILES"): 408 if get_bitbake_var("IMAGE_EFI_BOOT_FILES"):
296 for src_path, dst_path in cls.install_task: 409 for src_path, dst_path in cls.install_task:
@@ -312,6 +425,28 @@ class BootimgEFIPlugin(SourcePlugin):
312 for mod in [x for x in os.listdir(kernel_dir) if x.startswith("systemd-")]: 425 for mod in [x for x in os.listdir(kernel_dir) if x.startswith("systemd-")]:
313 cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[8:]) 426 cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[8:])
314 exec_cmd(cp_cmd, True) 427 exec_cmd(cp_cmd, True)
428 elif source_params['loader'] == 'uefi-kernel':
429 kernel = get_bitbake_var("KERNEL_IMAGETYPE")
430 if not kernel:
431 raise WicError("Empty KERNEL_IMAGETYPE %s\n" % target)
432 target = get_bitbake_var("TARGET_SYS")
433 if not target:
434 raise WicError("Unknown arch (TARGET_SYS) %s\n" % target)
435
436 if re.match("x86_64", target):
437 kernel_efi_image = "bootx64.efi"
438 elif re.match('i.86', target):
439 kernel_efi_image = "bootia32.efi"
440 elif re.match('aarch64', target):
441 kernel_efi_image = "bootaa64.efi"
442 elif re.match('arm', target):
443 kernel_efi_image = "bootarm.efi"
444 else:
445 raise WicError("UEFI stub kernel is incompatible with target %s" % target)
446
447 for mod in [x for x in os.listdir(kernel_dir) if x.startswith(kernel)]:
448 cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, kernel_efi_image)
449 exec_cmd(cp_cmd, True)
315 else: 450 else:
316 raise WicError("unrecognized bootimg-efi loader: %s" % 451 raise WicError("unrecognized bootimg-efi loader: %s" %
317 source_params['loader']) 452 source_params['loader'])
@@ -323,6 +458,11 @@ class BootimgEFIPlugin(SourcePlugin):
323 cp_cmd = "cp %s %s/" % (startup, hdddir) 458 cp_cmd = "cp %s %s/" % (startup, hdddir)
324 exec_cmd(cp_cmd, True) 459 exec_cmd(cp_cmd, True)
325 460
461 for paths in part.include_path or []:
462 for path in paths:
463 cp_cmd = "cp -r %s %s/" % (path, hdddir)
464 exec_cmd(cp_cmd, True)
465
326 du_cmd = "du -bks %s" % hdddir 466 du_cmd = "du -bks %s" % hdddir
327 out = exec_cmd(du_cmd) 467 out = exec_cmd(du_cmd)
328 blocks = int(out.split()[0]) 468 blocks = int(out.split()[0])
@@ -337,6 +477,13 @@ class BootimgEFIPlugin(SourcePlugin):
337 logger.debug("Added %d extra blocks to %s to get to %d total blocks", 477 logger.debug("Added %d extra blocks to %s to get to %d total blocks",
338 extra_blocks, part.mountpoint, blocks) 478 extra_blocks, part.mountpoint, blocks)
339 479
480 # required for compatibility with certain devices expecting file system
481 # block count to be equal to partition block count
482 if blocks < part.fixed_size:
483 blocks = part.fixed_size
484 logger.debug("Overriding %s to %d total blocks for compatibility",
485 part.mountpoint, blocks)
486
340 # dosfs image, created by mkdosfs 487 # dosfs image, created by mkdosfs
341 bootimg = "%s/boot.img" % cr_workdir 488 bootimg = "%s/boot.img" % cr_workdir
342 489
diff --git a/scripts/lib/wic/plugins/source/bootimg-partition.py b/scripts/lib/wic/plugins/source/bootimg-partition.py
index 5dbe2558d2..1071d1af3f 100644
--- a/scripts/lib/wic/plugins/source/bootimg-partition.py
+++ b/scripts/lib/wic/plugins/source/bootimg-partition.py
@@ -1,4 +1,6 @@
1# 1#
2# Copyright OpenEmbedded Contributors
3#
2# SPDX-License-Identifier: GPL-2.0-only 4# SPDX-License-Identifier: GPL-2.0-only
3# 5#
4# DESCRIPTION 6# DESCRIPTION
@@ -30,6 +32,7 @@ class BootimgPartitionPlugin(SourcePlugin):
30 """ 32 """
31 33
32 name = 'bootimg-partition' 34 name = 'bootimg-partition'
35 image_boot_files_var_name = 'IMAGE_BOOT_FILES'
33 36
34 @classmethod 37 @classmethod
35 def do_configure_partition(cls, part, source_params, cr, cr_workdir, 38 def do_configure_partition(cls, part, source_params, cr, cr_workdir,
@@ -54,12 +57,12 @@ class BootimgPartitionPlugin(SourcePlugin):
54 else: 57 else:
55 var = "" 58 var = ""
56 59
57 boot_files = get_bitbake_var("IMAGE_BOOT_FILES" + var) 60 boot_files = get_bitbake_var(cls.image_boot_files_var_name + var)
58 if boot_files is not None: 61 if boot_files is not None:
59 break 62 break
60 63
61 if boot_files is None: 64 if boot_files is None:
62 raise WicError('No boot files defined, IMAGE_BOOT_FILES unset for entry #%d' % part.lineno) 65 raise WicError('No boot files defined, %s unset for entry #%d' % (cls.image_boot_files_var_name, part.lineno))
63 66
64 logger.debug('Boot files: %s', boot_files) 67 logger.debug('Boot files: %s', boot_files)
65 68
@@ -110,7 +113,7 @@ class BootimgPartitionPlugin(SourcePlugin):
110 # Use a custom configuration for extlinux.conf 113 # Use a custom configuration for extlinux.conf
111 extlinux_conf = custom_cfg 114 extlinux_conf = custom_cfg
112 logger.debug("Using custom configuration file " 115 logger.debug("Using custom configuration file "
113 "%s for extlinux.cfg", configfile) 116 "%s for extlinux.conf", configfile)
114 else: 117 else:
115 raise WicError("configfile is specified but failed to " 118 raise WicError("configfile is specified but failed to "
116 "get it from %s." % configfile) 119 "get it from %s." % configfile)
diff --git a/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/scripts/lib/wic/plugins/source/bootimg-pcbios.py
index f2639e7004..a207a83530 100644
--- a/scripts/lib/wic/plugins/source/bootimg-pcbios.py
+++ b/scripts/lib/wic/plugins/source/bootimg-pcbios.py
@@ -122,7 +122,7 @@ class BootimgPcbiosPlugin(SourcePlugin):
122 syslinux_conf += "DEFAULT boot\n" 122 syslinux_conf += "DEFAULT boot\n"
123 syslinux_conf += "LABEL boot\n" 123 syslinux_conf += "LABEL boot\n"
124 124
125 kernel = "/vmlinuz" 125 kernel = "/" + get_bitbake_var("KERNEL_IMAGETYPE")
126 syslinux_conf += "KERNEL " + kernel + "\n" 126 syslinux_conf += "KERNEL " + kernel + "\n"
127 127
128 syslinux_conf += "APPEND label=boot root=%s %s\n" % \ 128 syslinux_conf += "APPEND label=boot root=%s %s\n" % \
@@ -155,8 +155,8 @@ class BootimgPcbiosPlugin(SourcePlugin):
155 kernel = "%s-%s.bin" % \ 155 kernel = "%s-%s.bin" % \
156 (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME")) 156 (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
157 157
158 cmds = ("install -m 0644 %s/%s %s/vmlinuz" % 158 cmds = ("install -m 0644 %s/%s %s/%s" %
159 (staging_kernel_dir, kernel, hdddir), 159 (staging_kernel_dir, kernel, hdddir, get_bitbake_var("KERNEL_IMAGETYPE")),
160 "install -m 444 %s/syslinux/ldlinux.sys %s/ldlinux.sys" % 160 "install -m 444 %s/syslinux/ldlinux.sys %s/ldlinux.sys" %
161 (bootimg_dir, hdddir), 161 (bootimg_dir, hdddir),
162 "install -m 0644 %s/syslinux/vesamenu.c32 %s/vesamenu.c32" % 162 "install -m 0644 %s/syslinux/vesamenu.c32 %s/vesamenu.c32" %
@@ -186,8 +186,10 @@ class BootimgPcbiosPlugin(SourcePlugin):
186 # dosfs image, created by mkdosfs 186 # dosfs image, created by mkdosfs
187 bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno) 187 bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno)
188 188
189 dosfs_cmd = "mkdosfs -n boot -i %s -S 512 -C %s %d" % \ 189 label = part.label if part.label else "boot"
190 (part.fsuuid, bootimg, blocks) 190
191 dosfs_cmd = "mkdosfs -n %s -i %s -S 512 -C %s %d" % \
192 (label, part.fsuuid, bootimg, blocks)
191 exec_native_cmd(dosfs_cmd, native_sysroot) 193 exec_native_cmd(dosfs_cmd, native_sysroot)
192 194
193 mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir) 195 mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir)
diff --git a/scripts/lib/wic/plugins/source/empty.py b/scripts/lib/wic/plugins/source/empty.py
index 041617d648..4178912377 100644
--- a/scripts/lib/wic/plugins/source/empty.py
+++ b/scripts/lib/wic/plugins/source/empty.py
@@ -1,4 +1,6 @@
1# 1#
2# Copyright OpenEmbedded Contributors
3#
2# SPDX-License-Identifier: MIT 4# SPDX-License-Identifier: MIT
3# 5#
4 6
@@ -7,9 +9,19 @@
7# To use it you must pass "empty" as argument for the "--source" parameter in 9# To use it you must pass "empty" as argument for the "--source" parameter in
8# the wks file. For example: 10# the wks file. For example:
9# part foo --source empty --ondisk sda --size="1024" --align 1024 11# part foo --source empty --ondisk sda --size="1024" --align 1024
12#
13# The plugin supports writing zeros to the start of the
14# partition. This is useful to overwrite old content like
15# filesystem signatures which may be re-recognized otherwise.
16# This feature can be enabled with
17# '--sourceparams="[fill|size=<N>[S|s|K|k|M|G]][,][bs=<N>[S|s|K|k|M|G]]"'
18# Conflicting or missing options throw errors.
10 19
11import logging 20import logging
21import os
12 22
23from wic import WicError
24from wic.ksparser import sizetype
13from wic.pluginbase import SourcePlugin 25from wic.pluginbase import SourcePlugin
14 26
15logger = logging.getLogger('wic') 27logger = logging.getLogger('wic')
@@ -17,6 +29,16 @@ logger = logging.getLogger('wic')
17class EmptyPartitionPlugin(SourcePlugin): 29class EmptyPartitionPlugin(SourcePlugin):
18 """ 30 """
19 Populate unformatted empty partition. 31 Populate unformatted empty partition.
32
33 The following sourceparams are supported:
34 - fill
35 Fill the entire partition with zeros. Requires '--fixed-size' option
36 to be set.
37 - size=<N>[S|s|K|k|M|G]
38 Set the first N bytes of the partition to zero. Default unit is 'K'.
39 - bs=<N>[S|s|K|k|M|G]
40 Write at most N bytes at a time during source file creation.
41 Defaults to '1M'. Default unit is 'K'.
20 """ 42 """
21 43
22 name = 'empty' 44 name = 'empty'
@@ -29,4 +51,39 @@ class EmptyPartitionPlugin(SourcePlugin):
29 Called to do the actual content population for a partition i.e. it 51 Called to do the actual content population for a partition i.e. it
30 'prepares' the partition to be incorporated into the image. 52 'prepares' the partition to be incorporated into the image.
31 """ 53 """
32 return 54 get_byte_count = sizetype('K', True)
55 size = 0
56
57 if 'fill' in source_params and 'size' in source_params:
58 raise WicError("Conflicting source parameters 'fill' and 'size' specified, exiting.")
59
60 # Set the size of the zeros to be written to the partition
61 if 'fill' in source_params:
62 if part.fixed_size == 0:
63 raise WicError("Source parameter 'fill' only works with the '--fixed-size' option, exiting.")
64 size = get_byte_count(part.fixed_size)
65 elif 'size' in source_params:
66 size = get_byte_count(source_params['size'])
67
68 if size == 0:
69 # Nothing to do, create empty partition
70 return
71
72 if 'bs' in source_params:
73 bs = get_byte_count(source_params['bs'])
74 else:
75 bs = get_byte_count('1M')
76
77 # Create a binary file of the requested size filled with zeros
78 source_file = os.path.join(cr_workdir, 'empty-plugin-zeros%s.bin' % part.lineno)
79 if not os.path.exists(os.path.dirname(source_file)):
80 os.makedirs(os.path.dirname(source_file))
81
82 quotient, remainder = divmod(size, bs)
83 with open(source_file, 'wb') as file:
84 for _ in range(quotient):
85 file.write(bytearray(bs))
86 file.write(bytearray(remainder))
87
88 part.size = (size + 1024 - 1) // 1024 # size in KB rounded up
89 part.source_file = source_file
diff --git a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
index afc9ea0f8f..607356ad13 100644
--- a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
+++ b/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
@@ -1,4 +1,6 @@
1# 1#
2# Copyright OpenEmbedded Contributors
3#
2# SPDX-License-Identifier: GPL-2.0-only 4# SPDX-License-Identifier: GPL-2.0-only
3# 5#
4# DESCRIPTION 6# DESCRIPTION
diff --git a/scripts/lib/wic/plugins/source/rawcopy.py b/scripts/lib/wic/plugins/source/rawcopy.py
index 3c4997d8ba..21903c2f23 100644
--- a/scripts/lib/wic/plugins/source/rawcopy.py
+++ b/scripts/lib/wic/plugins/source/rawcopy.py
@@ -1,9 +1,13 @@
1# 1#
2# Copyright OpenEmbedded Contributors
3#
2# SPDX-License-Identifier: GPL-2.0-only 4# SPDX-License-Identifier: GPL-2.0-only
3# 5#
4 6
5import logging 7import logging
6import os 8import os
9import signal
10import subprocess
7 11
8from wic import WicError 12from wic import WicError
9from wic.pluginbase import SourcePlugin 13from wic.pluginbase import SourcePlugin
@@ -21,6 +25,10 @@ class RawCopyPlugin(SourcePlugin):
21 25
22 @staticmethod 26 @staticmethod
23 def do_image_label(fstype, dst, label): 27 def do_image_label(fstype, dst, label):
28 # don't create label when fstype is none
29 if fstype == 'none':
30 return
31
24 if fstype.startswith('ext'): 32 if fstype.startswith('ext'):
25 cmd = 'tune2fs -L %s %s' % (label, dst) 33 cmd = 'tune2fs -L %s %s' % (label, dst)
26 elif fstype in ('msdos', 'vfat'): 34 elif fstype in ('msdos', 'vfat'):
@@ -29,15 +37,35 @@ class RawCopyPlugin(SourcePlugin):
29 cmd = 'btrfs filesystem label %s %s' % (dst, label) 37 cmd = 'btrfs filesystem label %s %s' % (dst, label)
30 elif fstype == 'swap': 38 elif fstype == 'swap':
31 cmd = 'mkswap -L %s %s' % (label, dst) 39 cmd = 'mkswap -L %s %s' % (label, dst)
32 elif fstype == 'squashfs': 40 elif fstype in ('squashfs', 'erofs'):
33 raise WicError("It's not possible to update a squashfs " 41 raise WicError("It's not possible to update a %s "
34 "filesystem label '%s'" % (label)) 42 "filesystem label '%s'" % (fstype, label))
35 else: 43 else:
36 raise WicError("Cannot update filesystem label: " 44 raise WicError("Cannot update filesystem label: "
37 "Unknown fstype: '%s'" % (fstype)) 45 "Unknown fstype: '%s'" % (fstype))
38 46
39 exec_cmd(cmd) 47 exec_cmd(cmd)
40 48
49 @staticmethod
50 def do_image_uncompression(src, dst, workdir):
51 def subprocess_setup():
52 # Python installs a SIGPIPE handler by default. This is usually not what
53 # non-Python subprocesses expect.
54 # SIGPIPE errors are known issues with gzip/bash
55 signal.signal(signal.SIGPIPE, signal.SIG_DFL)
56
57 extension = os.path.splitext(src)[1]
58 decompressor = {
59 ".bz2": "bzip2",
60 ".gz": "gzip",
61 ".xz": "xz",
62 ".zst": "zstd -f",
63 }.get(extension)
64 if not decompressor:
65 raise WicError("Not supported compressor filename extension: %s" % extension)
66 cmd = "%s -dc %s > %s" % (decompressor, src, dst)
67 subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=workdir)
68
41 @classmethod 69 @classmethod
42 def do_prepare_partition(cls, part, source_params, cr, cr_workdir, 70 def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
43 oe_builddir, bootimg_dir, kernel_dir, 71 oe_builddir, bootimg_dir, kernel_dir,
@@ -56,7 +84,13 @@ class RawCopyPlugin(SourcePlugin):
56 if 'file' not in source_params: 84 if 'file' not in source_params:
57 raise WicError("No file specified") 85 raise WicError("No file specified")
58 86
59 src = os.path.join(kernel_dir, source_params['file']) 87 if 'unpack' in source_params:
88 img = os.path.join(kernel_dir, source_params['file'])
89 src = os.path.join(cr_workdir, os.path.splitext(source_params['file'])[0])
90 RawCopyPlugin.do_image_uncompression(img, src, cr_workdir)
91 else:
92 src = os.path.join(kernel_dir, source_params['file'])
93
60 dst = os.path.join(cr_workdir, "%s.%s" % (os.path.basename(source_params['file']), part.lineno)) 94 dst = os.path.join(cr_workdir, "%s.%s" % (os.path.basename(source_params['file']), part.lineno))
61 95
62 if not os.path.exists(os.path.dirname(dst)): 96 if not os.path.exists(os.path.dirname(dst)):
diff --git a/scripts/lib/wic/plugins/source/rootfs.py b/scripts/lib/wic/plugins/source/rootfs.py
index 96d940a91d..c990143c0d 100644
--- a/scripts/lib/wic/plugins/source/rootfs.py
+++ b/scripts/lib/wic/plugins/source/rootfs.py
@@ -35,7 +35,7 @@ class RootfsPlugin(SourcePlugin):
35 @staticmethod 35 @staticmethod
36 def __validate_path(cmd, rootfs_dir, path): 36 def __validate_path(cmd, rootfs_dir, path):
37 if os.path.isabs(path): 37 if os.path.isabs(path):
38 logger.error("%s: Must be relative: %s" % (cmd, orig_path)) 38 logger.error("%s: Must be relative: %s" % (cmd, path))
39 sys.exit(1) 39 sys.exit(1)
40 40
41 # Disallow climbing outside of parent directory using '..', 41 # Disallow climbing outside of parent directory using '..',
@@ -43,14 +43,14 @@ class RootfsPlugin(SourcePlugin):
43 # directory, or modify a directory outside OpenEmbedded). 43 # directory, or modify a directory outside OpenEmbedded).
44 full_path = os.path.realpath(os.path.join(rootfs_dir, path)) 44 full_path = os.path.realpath(os.path.join(rootfs_dir, path))
45 if not full_path.startswith(os.path.realpath(rootfs_dir)): 45 if not full_path.startswith(os.path.realpath(rootfs_dir)):
46 logger.error("%s: Must point inside the rootfs:" % (cmd, path)) 46 logger.error("%s: Must point inside the rootfs: %s" % (cmd, path))
47 sys.exit(1) 47 sys.exit(1)
48 48
49 return full_path 49 return full_path
50 50
51 @staticmethod 51 @staticmethod
52 def __get_rootfs_dir(rootfs_dir): 52 def __get_rootfs_dir(rootfs_dir):
53 if os.path.isdir(rootfs_dir): 53 if rootfs_dir and os.path.isdir(rootfs_dir):
54 return os.path.realpath(rootfs_dir) 54 return os.path.realpath(rootfs_dir)
55 55
56 image_rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", rootfs_dir) 56 image_rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", rootfs_dir)
@@ -97,6 +97,9 @@ class RootfsPlugin(SourcePlugin):
97 part.has_fstab = os.path.exists(os.path.join(part.rootfs_dir, "etc/fstab")) 97 part.has_fstab = os.path.exists(os.path.join(part.rootfs_dir, "etc/fstab"))
98 pseudo_dir = os.path.join(part.rootfs_dir, "../pseudo") 98 pseudo_dir = os.path.join(part.rootfs_dir, "../pseudo")
99 if not os.path.lexists(pseudo_dir): 99 if not os.path.lexists(pseudo_dir):
100 pseudo_dir = os.path.join(cls.__get_rootfs_dir(None), '../pseudo')
101
102 if not os.path.lexists(pseudo_dir):
100 logger.warn("%s folder does not exist. " 103 logger.warn("%s folder does not exist. "
101 "Usernames and permissions will be invalid " % pseudo_dir) 104 "Usernames and permissions will be invalid " % pseudo_dir)
102 pseudo_dir = None 105 pseudo_dir = None
@@ -218,10 +221,10 @@ class RootfsPlugin(SourcePlugin):
218 # Update part.has_fstab here as fstab may have been added or 221 # Update part.has_fstab here as fstab may have been added or
219 # removed by the above modifications. 222 # removed by the above modifications.
220 part.has_fstab = os.path.exists(os.path.join(new_rootfs, "etc/fstab")) 223 part.has_fstab = os.path.exists(os.path.join(new_rootfs, "etc/fstab"))
221 if part.update_fstab_in_rootfs and part.has_fstab: 224 if part.update_fstab_in_rootfs and part.has_fstab and not part.no_fstab_update:
222 fstab_path = os.path.join(new_rootfs, "etc/fstab") 225 fstab_path = os.path.join(new_rootfs, "etc/fstab")
223 # Assume that fstab should always be owned by root with fixed permissions 226 # Assume that fstab should always be owned by root with fixed permissions
224 install_cmd = "install -m 0644 %s %s" % (part.updated_fstab_path, fstab_path) 227 install_cmd = "install -m 0644 -p %s %s" % (part.updated_fstab_path, fstab_path)
225 if new_pseudo: 228 if new_pseudo:
226 pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo) 229 pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo)
227 else: 230 else:
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2024-05-05 08:04:46 +0000